add post title to comment

2019-01-05 04:24:49 -06:00
parent 7b54b00c01
commit 690a0aa763
4 changed files with 154 additions and 0 deletions
@@ -22,6 +22,7 @@ type Comment struct {
 	Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
 	Pin       bool            `json:"pin,omitempty" bson:"pin,omitempty"`
 	Deleted   bool            `json:"delete,omitempty" bson:"delete"`
+	PostTitle string          `json:"title,omitempty" bson:"title"`
 }

 // Locator keeps site and url of the post
@@ -21,6 +21,7 @@ type DataStore struct {
 	AdminStore     admin.Store
 	MaxCommentSize int
 	MaxVotes       int
+	TitleExtractor TitleExtractor

 	// granular locks
 	scopedLocks struct {
@@ -58,6 +59,9 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		return "", errors.Wrap(err, "failed to prepare comment")
 	}

+	if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil {
+		comment.PostTitle = title
+	}
 	return s.Interface.Create(comment)
 }

@@ -0,0 +1,86 @@
+package service
+
+import (
+	"io"
+	"log"
+	"net/http"
+
+	"github.com/go-pkgz/rest/cache"
+	"github.com/pkg/errors"
+	"golang.org/x/net/html"
+)
+
+// TitleExtractor gets html title from remote page, cached
+type TitleExtractor struct {
+	client http.Client
+	cache  cache.LoadingCache
+}
+
+// NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache
+func NewTitleExtractor(client http.Client) *TitleExtractor {
+	res := TitleExtractor{
+		client: client,
+	}
+	var err error
+	res.cache, err = cache.NewMemoryCache(cache.MaxKeys(1000))
+	if err != nil {
+		log.Printf("[WARN] failed to make cache, %v", err)
+		res.cache = &cache.Nop{}
+	}
+	return &res
+}
+
+// Get page for url and return title
+func (t *TitleExtractor) Get(url string) (string, error) {
+
+	b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) {
+		resp, err := t.client.Get(url)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to load page %s", url)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			return nil, errors.Errorf("can't load page %s, code %d", url, resp.StatusCode)
+		}
+
+		title, ok := t.getTitle(resp.Body)
+		if !ok {
+			return nil, errors.Errorf("can't get title for %s", url)
+		}
+		return []byte(title), nil
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	return string(b), nil
+}
+
+// get title from body reader, traverse recursively
+func (t *TitleExtractor) getTitle(r io.Reader) (string, bool) {
+	doc, err := html.Parse(r)
+	if err != nil {
+		log.Printf("[WARN] can't get header, %+v", err)
+		return "", false
+	}
+	return t.traverse(doc)
+}
+
+func (t *TitleExtractor) isTitleElement(n *html.Node) bool {
+	return n.Type == html.ElementNode && n.Data == "title"
+}
+
+func (t *TitleExtractor) traverse(n *html.Node) (string, bool) {
+	if t.isTitleElement(n) {
+		return n.FirstChild.Data, true
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		result, ok := t.traverse(c)
+		if ok {
+			return result, ok
+		}
+	}
+	return "", false
+}
@@ -0,0 +1,63 @@
+package service
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTitle_GetTitle(t *testing.T) {
+
+	tbl := []struct {
+		page  string
+		ok    bool
+		title string
+	}{
+		{`<html><title>blah 123</title><body> 2222</body></html>`, true, "blah 123"},
+		{`<html><title>blah 123 `, true, "blah 123 "},
+		{`<html><body> 2222</body></html>`, false, ""},
+	}
+
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	for i, tt := range tbl {
+		t.Run(fmt.Sprintf("check-%d", i), func(t *testing.T) {
+			title, ok := ex.getTitle(strings.NewReader(tt.page))
+			assert.Equal(t, tt.ok, ok)
+			assert.Equal(t, tt.title, title)
+		})
+	}
+}
+
+func TestTitle_Get(t *testing.T) {
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.String() == "/good" {
+			atomic.AddInt32(&hits, 1)
+			w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	title, err := ex.Get(ts.URL + "/good")
+	require.Nil(t, err)
+	assert.Equal(t, "blah 123", title)
+
+	_, err = ex.Get(ts.URL + "/bad")
+	require.NotNil(t, err)
+
+	for i := 0; i < 100; i++ {
+		title, err := ex.Get(ts.URL + "/good")
+		require.Nil(t, err)
+		assert.Equal(t, "blah 123", title)
+	}
+	assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
+}