diff --git a/backend/app/cmd/cleanup.go b/backend/app/cmd/cleanup.go
index cb06dad9..0371b1bb 100644
--- a/backend/app/cmd/cleanup.go
+++ b/backend/app/cmd/cleanup.go
@@ -22,6 +22,7 @@ type CleanupCommand struct {
BadWords []string `short:"w" long:"bword" description:"bad word(s)"`
BadUsers []string `short:"u" long:"buser" description:"bad user(s)"`
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+ SetTitle bool `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
CommonOpts
}
@@ -47,24 +48,51 @@ func (cc *CleanupCommand) Execute(args []string) error {
if e != nil {
continue
}
- for _, comment := range comments {
- totalComments++
- spam, score := cc.isSpam(comment)
- if spam {
- spamComments++
- if !cc.Dry {
- if err = cc.deleteComment(comment); err != nil {
- log.Printf("[WARN] can't remove comment, %v", err)
- }
- }
- comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
- log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
+ totalComments += len(comments)
+ if cc.SetTitle {
+ cc.procTitles(comments)
+ } else {
+ spamComments += cc.procSpam(comments)
+
+ }
+ }
+
+ msg := fmt.Sprintf("comments=%d, spam=%d", totalComments, spamComments)
+ if cc.SetTitle {
+ msg = fmt.Sprintf("comments=%d", totalComments)
+ }
+
+ log.Printf("[INFO] completed, %s", msg)
+ return err
+}
+
+func (cc *CleanupCommand) procSpam(comments []store.Comment) int {
+ spamComments := 0
+ for _, comment := range comments {
+ spam, score := cc.isSpam(comment)
+ if spam {
+ spamComments++
+ if !cc.Dry {
+ if err := cc.deleteComment(comment); err != nil {
+ log.Printf("[WARN] can't remove comment, %v", err)
+ }
+ }
+ comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
+ log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
+ }
+ }
+ return spamComments
+}
+
+func (cc *CleanupCommand) procTitles(comments []store.Comment) {
+ for _, comment := range comments {
+ if !cc.Dry {
+ if err := cc.setTitle(comment); err != nil {
+ log.Printf("[WARN] can't set title for comment, %v", err)
}
}
}
- log.Printf("[INFO] comments=%d, spam=%d", totalComments, spamComments)
- return err
}
// get list of posts in from/to represented as yyyymmdd. this is [from-to] inclusive
@@ -180,6 +208,28 @@ func (cc *CleanupCommand) deleteComment(c store.Comment) error {
return nil
}
+// setTitle with PUT /admin/title/{id}?site=siteID&url=post-url
+func (cc *CleanupCommand) setTitle(c store.Comment) error {
+
+ titleURL := fmt.Sprintf("%s/api/v1/admin/title/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
+ req, err := http.NewRequest("PUT", titleURL, nil)
+ if err != nil {
+ return errors.Wrapf(err, "failed to make title request for comment %s, %s", c.ID, c.Locator.URL)
+ }
+ req.SetBasicAuth("admin", cc.AdminPasswd)
+
+ client := http.Client{}
+ r, err := client.Do(req)
+ if err != nil {
+ return errors.Wrapf(err, "title request failed for comment %s, %s", c.ID, c.Locator.URL)
+ }
+ defer func() { _ = r.Body.Close() }()
+ if r.StatusCode != http.StatusOK {
+ return errors.Errorf("title request failed with status %s", r.Status)
+ }
+ return nil
+}
+
// isSpam calculates spam's probability as a score
func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
diff --git a/backend/app/cmd/cleanup_test.go b/backend/app/cmd/cleanup_test.go
index 1603ab40..3514abff 100644
--- a/backend/app/cmd/cleanup_test.go
+++ b/backend/app/cmd/cleanup_test.go
@@ -106,7 +106,7 @@ func TestCleanup_listComments(t *testing.T) {
assert.Equal(t, 0, len(comments))
}
-func TestCleanup_Execute(t *testing.T) {
+func TestCleanup_ExecuteSpam(t *testing.T) {
cleaned := cleanedComments{}
r := chi.NewRouter()
cleanupRoutes(t, r, &cleaned)
@@ -125,6 +125,24 @@ func TestCleanup_Execute(t *testing.T) {
assert.Equal(t, []string{"/api/v1/admin/comment/1", "/api/v1/admin/comment/3", "/api/v1/admin/comment/11"}, cleaned.ids)
}
+func TestCleanup_ExecuteTitle(t *testing.T) {
+ titledComments := cleanedComments{}
+ r := chi.NewRouter()
+ cleanupRoutes(t, r, &titledComments)
+ ts := httptest.NewServer(r)
+ defer ts.Close()
+
+ cmd := CleanupCommand{}
+ cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+ p := flags.NewParser(&cmd, flags.Default)
+ _, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
+ require.Nil(t, err)
+ err = cmd.Execute(nil)
+ assert.NoError(t, err)
+ t.Logf("set titles for %+v", titledComments.ids)
+ assert.Equal(t, []string{"/api/v1/admin/title/1", "/api/v1/admin/title/2", "/api/v1/admin/title/3", "/api/v1/admin/title/11"}, titledComments.ids)
+}
+
func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, "GET", r.Method)
@@ -184,4 +202,13 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
c.ids = append(c.ids, r.URL.Path)
c.lock.Unlock()
}))
+
+ r.HandleFunc("/api/v1/admin/title/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ require.Equal(t, "PUT", r.Method)
+ t.Log("title for ", r.URL.Path)
+ c.lock.Lock()
+ c.ids = append(c.ids, r.URL.Path)
+ c.lock.Unlock()
+ }))
+
}
diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go
index b4a60988..4cd0f420 100644
--- a/backend/app/rest/api/admin.go
+++ b/backend/app/rest/api/admin.go
@@ -40,6 +40,7 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
router.Put("/pin/{id}", a.setPinCtrl)
router.Get("/blocked", a.blockedUsersCtrl)
router.Put("/readonly", a.setReadOnlyCtrl)
+ router.Put("/title/{id}", a.setTitleCtrl)
a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
@@ -191,6 +192,23 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, R.JSON{"locator": locator, "read-only": roStatus})
}
+// PUT /title/{id}?site=siteID&url=post-url - set comment PostTitle to page's title
+func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
+ id := chi.URLParam(r, "id")
+ locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
+
+ c, err := a.dataService.SetTitle(locator, id)
+ if err != nil {
+ rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+ return
+ }
+ log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
+
+ a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
+ render.Status(r, http.StatusOK)
+ render.JSON(w, r, R.JSON{"id": id, "locator": locator})
+}
+
// PUT /verify?site=siteID&url=post-url&ro=1 - set or reset read-only status for the post
func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
userID := chi.URLParam(r, "userid")
diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go
index 6bbde081..1cc7ec45 100644
--- a/backend/app/rest/api/admin_test.go
+++ b/backend/app/rest/api/admin_test.go
@@ -7,6 +7,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
+ "net/http/httptest"
"os"
"strings"
"testing"
@@ -15,6 +16,7 @@ import (
jwt "github.com/dgrijalva/jwt-go"
"github.com/go-pkgz/auth/token"
R "github.com/go-pkgz/rest"
+ "github.com/umputun/remark/backend/app/store/service"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -52,6 +54,49 @@ func TestAdmin_Delete(t *testing.T) {
assert.True(t, cr.Deleted)
}
+func TestAdmin_Title(t *testing.T) {
+ ts, srv, teardown := startupT(t)
+ defer teardown()
+
+ srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
+ tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.String() == "/post1" {
+ w.Write([]byte("post1 blah 123 2222"))
+ return
+ }
+ if r.URL.String() == "/post2" {
+ w.Write([]byte("post2 blah 123 2222"))
+ return
+ }
+ w.WriteHeader(404)
+ }))
+ defer tss.Close()
+
+ c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
+ Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post1"}}
+ c2 := store.Comment{Text: "test test #2", User: store.User{ID: "id", Name: "name"}, ParentID: "p1",
+ Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post2"}}
+
+ id1 := addComment(t, c1, ts)
+ addComment(t, c2, ts)
+
+ client := http.Client{}
+ req, err := http.NewRequest(http.MethodPut,
+ fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
+ assert.Nil(t, err)
+ req.SetBasicAuth("admin", "password")
+ resp, err := client.Do(req)
+ require.Nil(t, err)
+ assert.Equal(t, 200, resp.StatusCode)
+
+ body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
+ require.Equal(t, 200, code)
+ cr := store.Comment{}
+ err = json.Unmarshal([]byte(body), &cr)
+ assert.Nil(t, err)
+ assert.Equal(t, "post1 blah 123", cr.PostTitle)
+}
+
func TestAdmin_DeleteUser(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 5c37ecf8..4b9590e6 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -196,6 +196,26 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
return comment, err
}
+func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error) {
+ if s.TitleExtractor == nil {
+ return comment, errors.New("no title extractor")
+ }
+
+ comment, err = s.Get(locator, commentID)
+ if err != nil {
+ return comment, err
+ }
+
+ // set title, overwrite the current one
+ title, e := s.TitleExtractor.Get(comment.Locator.URL)
+ if e != nil {
+ return comment, err
+ }
+ comment.PostTitle = title
+ err = s.Put(locator, comment)
+ return comment, err
+}
+
// Counts returns postID+count list for given comments
func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
res := []store.PostInfo{}
diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go
index da6a0ef0..f4292ba9 100644
--- a/backend/app/store/service/service_test.go
+++ b/backend/app/store/service/service_test.go
@@ -4,13 +4,15 @@ import (
"fmt"
"math/rand"
"net/http"
+ "net/http/httptest"
"os"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
- bolt "github.com/coreos/bbolt"
+ "github.com/coreos/bbolt"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -103,6 +105,56 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
assert.Equal(t, "post blah", res.PostTitle, "keep comment title")
}
+func TestService_SetTitle(t *testing.T) {
+ defer os.Remove(testDb)
+
+ var titleEnable int32
+ tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if atomic.LoadInt32(&titleEnable) == 0 {
+ w.WriteHeader(404)
+ }
+ if r.URL.String() == "/post1" {
+ w.Write([]byte("post1 blah 123 2222"))
+ return
+ }
+ if r.URL.String() == "/post2" {
+ w.Write([]byte("post2 blah 123 2222"))
+ return
+ }
+ w.WriteHeader(404)
+ }))
+ defer tss.Close()
+
+ ks := admin.NewStaticKeyStore("secret 123")
+ b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
+ TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+ comment := store.Comment{
+ Text: "text",
+ Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
+ Votes: map[string]bool{"u1": true, "u2": false},
+ User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
+ Locator: store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"},
+ }
+
+ id, err := b.Create(comment)
+ assert.NoError(t, err)
+ assert.True(t, id != "", id)
+
+ res, err := b.Get(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+ assert.NoError(t, err)
+ t.Logf("%+v", res)
+ assert.Equal(t, "", res.PostTitle)
+
+ atomic.StoreInt32(&titleEnable, 1)
+ c, err := b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+ require.NoError(t, err)
+ assert.Equal(t, "post1 blah 123", c.PostTitle)
+
+ b = DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+ _, err = b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+ require.EqualError(t, err, "no title extractor")
+}
+
func TestService_Vote(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}