add cleanup tests

2018-12-18 21:57:46 -06:00
parent c393359511
commit 7560e98560
2 changed files with 200 additions and 12 deletions
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"math"
 	"net/http"
 	"strings"
 	"time"
@@ -24,6 +25,7 @@ type CleanupCommand struct {
 }

 // Execute runs cleanup with CleanupCommand parameters, entry point for "cleanup" command
+// This command uses provided flags to detect and remove junk comments
 func (cc *CleanupCommand) Execute(args []string) error {
 	log.Printf("[INFO] cleanup for site %s", cc.Site)

@@ -31,7 +33,6 @@ func (cc *CleanupCommand) Execute(args []string) error {
 	if err != nil {
 		return errors.Wrap(err, "can't get posts")
 	}
-
 	log.Printf("[DEBUG] got %d posts", len(posts))

 	totalComments, spamComments := 0, 0
@@ -45,12 +46,14 @@ func (cc *CleanupCommand) Execute(args []string) error {
 			spam, score := cc.isSpam(comment)
 			if spam {
 				spamComments++
-				log.Printf("[SPAM] %+v [%.0f]", comment, score)
 				if !cc.Dry {
 					if err = cc.deleteComment(comment); err != nil {
 						log.Printf("[WARN] can't remove comment, %v", err)
 					}
 				}
+				comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
+				log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
+
 			}
 		}
 	}
@@ -58,7 +61,7 @@ func (cc *CleanupCommand) Execute(args []string) error {
 	return err
 }

-// get list of posts in from/to represented as yyyymmdd
+// get list of posts in from/to represented as yyyymmdd. this is [from-to] inclusive
 func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, error) {
 	posts, err := cc.listPosts()
 	if err != nil {
@@ -84,7 +87,7 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err

 	var filteredList []store.PostInfo
 	for _, postInfo := range posts {
-		if postInfo.FirstTS.After(from) && postInfo.LastTS.Before(to) {
+		if postInfo.FirstTS.After(from) && postInfo.LastTS.Before(to.AddDate(0, 0, 1)) {
 			filteredList = append(filteredList, postInfo)
 		}
 	}
@@ -175,7 +178,7 @@ func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
 		res := 0.0
 		for _, w := range cc.BadWords {
 			if strings.Contains(txt, w) {
-				res += 0.20
+				res += 0.25
 			}
 			if res > 1 {
 				return 1
@@ -200,19 +203,19 @@ func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
 		return false, 0
 	}

-	score += 50 * badWord(comment.Text) // up to 50, 5 bad words will reach max
+	score += 50 * badWord(comment.Text) // up to 50, 4 bad words will reach max

 	if hasBadUser(comment.User.ID) { // predefined list of bad user substrings
 		score += 10
 	}

 	if comment.Score == 0 { // most of spam comments with 0 score
-		score += 10
+		score += 20
 	}

 	// any link inside
 	if strings.Contains(comment.Text, "http:") || strings.Contains(comment.Text, "https:") {
-		score += 20
+		score += 10
 	}

 	// 5 or more links
@@ -220,10 +223,8 @@ func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
 		score += 10
 	}

-	// any score probably not for spam
-	if comment.Score != 0 {
-		score -= 30
-	}
+	score = math.Max(score, 0)
+	score = math.Min(score, 100)

 	return score > 50, score
 }
@@ -0,0 +1,187 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gorilla/mux"
+	flags "github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+type cleanedComments struct {
+	ids  []string
+	lock sync.Mutex
+}
+
+func TestCleanup_IsSpam(t *testing.T) {
+	cc := CleanupCommand{
+		BadWords: []string{"bad1", "bad2", "very bad", "xyz"},
+		BadUsers: []string{"bu_"},
+	}
+
+	tbl := []struct {
+		text      string
+		user      string
+		score     int
+		isSpam    bool
+		spamScore float64
+		name      string
+	}{
+		{"", "", 1, false, 0, "empty passes"},
+		{"one very bad two blah bad1 bad2 http://xyz.com", "bu_user", 0, true, 90, "3badwords link 0score baduser"},
+		{"one very bad two blah bad1 bad2", "bu_user", 0, true, 67.5, "3 bad words 1score baduser"},
+		{"bad1 bad2 xyz very bad", "bu_user", 0, true, 80, "4badwords 0score baduser"},
+		{"bad1 bad2 xyz very bad", "user", 0, true, 70, "4badwords 0score"},
+		{"bad1 bad2 xyz very bad", "user", 1, false, 50, "4badwords 1score"},
+		{"bad1 test 12345", "user", 0, false, 32.5, "1badwords 0score"},
+	}
+
+	for n, tt := range tbl {
+		checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
+		t.Run(checkName, func(t *testing.T) {
+			c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
+			c.User.ID = tt.user
+			r, score := cc.isSpam(c)
+			assert.Equal(t, tt.isSpam, r)
+			assert.InDelta(t, tt.spamScore, score, 0.01)
+		})
+	}
+}
+
+func TestCleanup_postsInRange(t *testing.T) {
+
+	r := mux.NewRouter()
+	cleanupRoutes(t, r, nil)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_"})
+	require.Nil(t, err)
+	posts, err := cmd.postsInRange("20181218", "20181219")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(posts))
+
+	posts, err = cmd.postsInRange("", "")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(posts))
+
+	_, err = cmd.postsInRange("xxx", "yyy")
+	assert.NotNil(t, err)
+}
+
+func TestCleanup_listComments(t *testing.T) {
+	r := mux.NewRouter()
+	cleanupRoutes(t, r, nil)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_"})
+	require.Nil(t, err)
+
+	comments, err := cmd.listComments("http://test.com/post1")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(comments))
+
+	comments, err = cmd.listComments("http://test.com/post2")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+
+	comments, err = cmd.listComments("http://test.com/post-bad")
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+}
+
+func TestCleanup_Execute(t *testing.T) {
+	cleaned := cleanedComments{}
+	r := mux.NewRouter()
+	cleanupRoutes(t, r, &cleaned)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_",
+		"--from=20181217", "--to=20181218"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	t.Logf("deleted %+v", cleaned.ids)
+	assert.Equal(t, []string{"/api/v1/admin/comment/1", "/api/v1/admin/comment/3", "/api/v1/admin/comment/11"}, cleaned.ids)
+}
+
+func cleanupRoutes(t *testing.T, r *mux.Router, c *cleanedComments) {
+	r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
+		list := []store.PostInfo{
+			{
+				URL:     "http://test.com/post1",
+				FirstTS: time.Date(2018, 12, 17, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 17, 10, 30, 0, 0, time.Local),
+			},
+			{
+				URL:     "http://test.com/post2",
+				FirstTS: time.Date(2018, 12, 18, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 18, 10, 30, 0, 0, time.Local),
+			},
+			{
+				URL:     "http://test.com/post3",
+				FirstTS: time.Date(2018, 12, 19, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 19, 10, 30, 0, 0, time.Local),
+			},
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(list))
+	}))
+
+	r.HandleFunc("/api/v1/find", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		require.Equal(t, "remark", r.URL.Query().Get("site"))
+		require.Equal(t, "plain", r.URL.Query().Get("format"))
+
+		commentsWithInfo := struct {
+			Comments []store.Comment `json:"comments"`
+			Info     store.PostInfo  `json:"info,omitempty"`
+		}{}
+
+		switch r.URL.Query().Get("url") {
+		case "http://test.com/post1":
+			commentsWithInfo.Comments = []store.Comment{
+				{ID: "1", Text: "one very bad two blah bad1 bad2 http://xyz.com", Score: 0, User: store.User{ID: "bu_user"}},
+				{ID: "2", Text: "good one http://xyz.com", Score: 1, User: store.User{ID: "bu_user"}},
+				{ID: "3", Text: "http://xyz.com bad1 bad2", Score: 0, User: store.User{ID: "user"}},
+			}
+		case "http://test.com/post2":
+			commentsWithInfo.Comments = []store.Comment{
+				{ID: "11", Text: "one very bad two blah bad1 bad2 http://xyz.com", Score: 0, User: store.User{ID: "bu_user"}},
+			}
+		case "http://test.com/post3":
+			commentsWithInfo.Comments = []store.Comment{}
+		}
+
+		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
+	}))
+
+	r.HandleFunc("/api/v1/admin/comment/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "DELETE", r.Method)
+		t.Log("delete ", r.URL.Path)
+		c.lock.Lock()
+		c.ids = append(c.ids, r.URL.Path)
+		c.lock.Unlock()
+	}))
+}