diff --git a/backend/app/store/comment.go b/backend/app/store/comment.go index a3b02557..c62eeeba 100644 --- a/backend/app/store/comment.go +++ b/backend/app/store/comment.go @@ -22,6 +22,7 @@ type Comment struct { Edit *Edit `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response Pin bool `json:"pin,omitempty" bson:"pin,omitempty"` Deleted bool `json:"delete,omitempty" bson:"delete"` + PostTitle string `json:"title,omitempty" bson:"title"` } // Locator keeps site and url of the post diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 3bb83e0d..3ffbdb27 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -21,6 +21,7 @@ type DataStore struct { AdminStore admin.Store MaxCommentSize int MaxVotes int + TitleExtractor TitleExtractor // granular locks scopedLocks struct { @@ -58,6 +59,9 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) return "", errors.Wrap(err, "failed to prepare comment") } + if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil { + comment.PostTitle = title + } return s.Interface.Create(comment) } diff --git a/backend/app/store/service/title.go b/backend/app/store/service/title.go new file mode 100644 index 00000000..5145ff58 --- /dev/null +++ b/backend/app/store/service/title.go @@ -0,0 +1,86 @@ +package service + +import ( + "io" + "log" + "net/http" + + "github.com/go-pkgz/rest/cache" + "github.com/pkg/errors" + "golang.org/x/net/html" +) + +// TitleExtractor gets html title from remote page, cached +type TitleExtractor struct { + client http.Client + cache cache.LoadingCache +} + +// NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache +func NewTitleExtractor(client http.Client) *TitleExtractor { + res := TitleExtractor{ + client: client, + } + var err error + res.cache, err = cache.NewMemoryCache(cache.MaxKeys(1000)) + if err != nil { + log.Printf("[WARN] failed to make cache, %v", err) + res.cache = &cache.Nop{} + } + return &res +} + +// Get page for url and return title +func (t *TitleExtractor) Get(url string) (string, error) { + + b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) { + resp, err := t.client.Get(url) + if err != nil { + return nil, errors.Wrapf(err, "failed to load page %s", url) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, errors.Errorf("can't load page %s, code %d", url, resp.StatusCode) + } + + title, ok := t.getTitle(resp.Body) + if !ok { + return nil, errors.Errorf("can't get title for %s", url) + } + return []byte(title), nil + }) + + if err != nil { + return "", err + } + + return string(b), nil +} + +// get title from body reader, traverse recursively +func (t *TitleExtractor) getTitle(r io.Reader) (string, bool) { + doc, err := html.Parse(r) + if err != nil { + log.Printf("[WARN] can't get header, %+v", err) + return "", false + } + return t.traverse(doc) +} + +func (t *TitleExtractor) isTitleElement(n *html.Node) bool { + return n.Type == html.ElementNode && n.Data == "title" +} + +func (t *TitleExtractor) traverse(n *html.Node) (string, bool) { + if t.isTitleElement(n) { + return n.FirstChild.Data, true + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + result, ok := t.traverse(c) + if ok { + return result, ok + } + } + return "", false +} diff --git a/backend/app/store/service/title_test.go b/backend/app/store/service/title_test.go new file mode 100644 index 00000000..a5a35c74 --- /dev/null +++ b/backend/app/store/service/title_test.go @@ -0,0 +1,63 @@ +package service + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTitle_GetTitle(t *testing.T) { + + tbl := []struct { + page string + ok bool + title string + }{ + {`