add rest and batch (cleanup --title) support for title's #241

backend/app/cmd/cleanup.go

@@ -22,6 +22,7 @@ type CleanupCommand struct {
 	BadWords    []string `short:"w" long:"bword" description:"bad word(s)"`
 	BadUsers    []string `short:"u" long:"buser" description:"bad user(s)"`
 	AdminPasswd string   `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	SetTitle    bool     `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
 	CommonOpts
 }
 
@@ -47,24 +48,51 @@ func (cc *CleanupCommand) Execute(args []string) error {
 		if e != nil {
 			continue
 		}
-		for _, comment := range comments {
-			totalComments++
-			spam, score := cc.isSpam(comment)
-			if spam {
-				spamComments++
-				if !cc.Dry {
-					if err = cc.deleteComment(comment); err != nil {
-						log.Printf("[WARN] can't remove comment, %v", err)
-					}
-				}
-				comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
-				log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
+		totalComments += len(comments)
 
+		if cc.SetTitle {
+			cc.procTitles(comments)
+		} else {
+			spamComments += cc.procSpam(comments)
+
+		}
+	}
+
+	msg := fmt.Sprintf("comments=%d, spam=%d", totalComments, spamComments)
+	if cc.SetTitle {
+		msg = fmt.Sprintf("comments=%d", totalComments)
+	}
+
+	log.Printf("[INFO] completed, %s", msg)
+	return err
+}
+
+func (cc *CleanupCommand) procSpam(comments []store.Comment) int {
+	spamComments := 0
+	for _, comment := range comments {
+		spam, score := cc.isSpam(comment)
+		if spam {
+			spamComments++
+			if !cc.Dry {
+				if err := cc.deleteComment(comment); err != nil {
+					log.Printf("[WARN] can't remove comment, %v", err)
+				}
+			}
+			comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
+			log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
+		}
+	}
+	return spamComments
+}
+
+func (cc *CleanupCommand) procTitles(comments []store.Comment) {
+	for _, comment := range comments {
+		if !cc.Dry {
+			if err := cc.setTitle(comment); err != nil {
+				log.Printf("[WARN] can't set title for comment, %v", err)
 			}
 		}
 	}
-	log.Printf("[INFO] comments=%d, spam=%d", totalComments, spamComments)
-	return err
 }
 
 // get list of posts in from/to represented as yyyymmdd. this is [from-to] inclusive
@@ -180,6 +208,28 @@ func (cc *CleanupCommand) deleteComment(c store.Comment) error {
 	return nil
 }
 
+// setTitle with PUT /admin/title/{id}?site=siteID&url=post-url
+func (cc *CleanupCommand) setTitle(c store.Comment) error {
+
+	titleURL := fmt.Sprintf("%s/api/v1/admin/title/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
+	req, err := http.NewRequest("PUT", titleURL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "failed to make title request for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	req.SetBasicAuth("admin", cc.AdminPasswd)
+
+	client := http.Client{}
+	r, err := client.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "title request failed for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	defer func() { _ = r.Body.Close() }()
+	if r.StatusCode != http.StatusOK {
+		return errors.Errorf("title request failed with status %s", r.Status)
+	}
+	return nil
+}
+
 // isSpam calculates spam's probability as a score
 func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
 

backend/app/cmd/cleanup_test.go

@@ -106,7 +106,7 @@ func TestCleanup_listComments(t *testing.T) {
 	assert.Equal(t, 0, len(comments))
 }
 
-func TestCleanup_Execute(t *testing.T) {
+func TestCleanup_ExecuteSpam(t *testing.T) {
 	cleaned := cleanedComments{}
 	r := chi.NewRouter()
 	cleanupRoutes(t, r, &cleaned)
@@ -125,6 +125,24 @@ func TestCleanup_Execute(t *testing.T) {
 	assert.Equal(t, []string{"/api/v1/admin/comment/1", "/api/v1/admin/comment/3", "/api/v1/admin/comment/11"}, cleaned.ids)
 }
 
+func TestCleanup_ExecuteTitle(t *testing.T) {
+	titledComments := cleanedComments{}
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, &titledComments)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	t.Logf("set titles for %+v", titledComments.ids)
+	assert.Equal(t, []string{"/api/v1/admin/title/1", "/api/v1/admin/title/2", "/api/v1/admin/title/3", "/api/v1/admin/title/11"}, titledComments.ids)
+}
+
 func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 	r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
@@ -184,4 +202,13 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
 	}))
+
+	r.HandleFunc("/api/v1/admin/title/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "PUT", r.Method)
+		t.Log("title for ", r.URL.Path)
+		c.lock.Lock()
+		c.ids = append(c.ids, r.URL.Path)
+		c.lock.Unlock()
+	}))
+
 }

backend/app/rest/api/admin.go

@@ -40,6 +40,7 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
 	router.Put("/pin/{id}", a.setPinCtrl)
 	router.Get("/blocked", a.blockedUsersCtrl)
 	router.Put("/readonly", a.setReadOnlyCtrl)
+	router.Put("/title/{id}", a.setTitleCtrl)
 
 	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
 
@@ -191,6 +192,23 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"locator": locator, "read-only": roStatus})
 }
 
+// PUT /title/{id}?site=siteID&url=post-url - set comment PostTitle to page's title
+func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
+	id := chi.URLParam(r, "id")
+	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
+
+	c, err := a.dataService.SetTitle(locator, id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+		return
+	}
+	log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
+
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, R.JSON{"id": id, "locator": locator})
+}
+
 // PUT /verify?site=siteID&url=post-url&ro=1 - set or reset read-only status for the post
 func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 	userID := chi.URLParam(r, "userid")

backend/app/rest/api/admin_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
@@ -15,6 +16,7 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
 	R "github.com/go-pkgz/rest"
+	"github.com/umputun/remark/backend/app/store/service"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -52,6 +54,49 @@ func TestAdmin_Delete(t *testing.T) {
 	assert.True(t, cr.Deleted)
 }
 
+func TestAdmin_Title(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
+	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.String() == "/post1" {
+			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			return
+		}
+		if r.URL.String() == "/post2" {
+			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			return
+		}
+		w.WriteHeader(404)
+	}))
+	defer tss.Close()
+
+	c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
+		Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post1"}}
+	c2 := store.Comment{Text: "test test #2", User: store.User{ID: "id", Name: "name"}, ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post2"}}
+
+	id1 := addComment(t, c1, ts)
+	addComment(t, c2, ts)
+
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+
+	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
+	require.Equal(t, 200, code)
+	cr := store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, "post1 blah 123", cr.PostTitle)
+}
+
 func TestAdmin_DeleteUser(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()

backend/app/store/service/service.go

@@ -196,6 +196,26 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 	return comment, err
 }
 
+func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error) {
+	if s.TitleExtractor == nil {
+		return comment, errors.New("no title extractor")
+	}
+
+	comment, err = s.Get(locator, commentID)
+	if err != nil {
+		return comment, err
+	}
+
+	// set title, overwrite the current one
+	title, e := s.TitleExtractor.Get(comment.Locator.URL)
+	if e != nil {
+		return comment, err
+	}
+	comment.PostTitle = title
+	err = s.Put(locator, comment)
+	return comment, err
+}
+
 // Counts returns postID+count list for given comments
 func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
 	res := []store.PostInfo{}

backend/app/store/service/service_test.go

@@ -4,13 +4,15 @@ import (
 	"fmt"
 	"math/rand"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
+	"github.com/coreos/bbolt"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -103,6 +105,56 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 	assert.Equal(t, "post blah", res.PostTitle, "keep comment title")
 }
 
+func TestService_SetTitle(t *testing.T) {
+	defer os.Remove(testDb)
+
+	var titleEnable int32
+	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if atomic.LoadInt32(&titleEnable) == 0 {
+			w.WriteHeader(404)
+		}
+		if r.URL.String() == "/post1" {
+			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			return
+		}
+		if r.URL.String() == "/post2" {
+			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			return
+		}
+		w.WriteHeader(404)
+	}))
+	defer tss.Close()
+
+	ks := admin.NewStaticKeyStore("secret 123")
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+	comment := store.Comment{
+		Text:      "text",
+		Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
+		Votes:     map[string]bool{"u1": true, "u2": false},
+		User:      store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
+		Locator:   store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"},
+	}
+
+	id, err := b.Create(comment)
+	assert.NoError(t, err)
+	assert.True(t, id != "", id)
+
+	res, err := b.Get(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+	assert.NoError(t, err)
+	t.Logf("%+v", res)
+	assert.Equal(t, "", res.PostTitle)
+
+	atomic.StoreInt32(&titleEnable, 1)
+	c, err := b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+	require.NoError(t, err)
+	assert.Equal(t, "post1 blah 123", c.PostTitle)
+
+	b = DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+	_, err = b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+	require.EqualError(t, err, "no title extractor")
+}
+
 func TestService_Vote(t *testing.T) {
 	defer os.Remove(testDb)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}