make title extractor optional

backend/app/cmd/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
@@ -214,6 +215,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		AdminStore:     adminStore,
 		MaxCommentSize: s.MaxCommentSize,
 		MaxVotes:       s.MaxVotes,
+		TitleExtractor: service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 	}
 
 	loadingCache, err := s.makeCache()

backend/app/cmd/server_test.go

@@ -316,7 +316,7 @@ func Test_ACMEEmail(t *testing.T) {
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 2500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx := prepServerApp(t, 10000*time.Millisecond, func(o ServerCommand) ServerCommand {
 		o.Port = 18080
 		return o
 	})
@@ -347,7 +347,7 @@ func TestServerAuthHooks(t *testing.T) {
 	// add comment
 	client := http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
-		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
 	req.Header.Set("X-JWT", tk)
 	require.Nil(t, err)
 	resp, err := client.Do(req)

backend/app/rest/api/rest_private.go

@@ -34,7 +34,6 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	user := rest.MustGetUserInfo(r)
-	log.Printf("[DEBUG] create comment %+v", comment)
 
 	comment.PrepareUntrusted() // clean all fields user not supposed to set
 	comment.User = user
@@ -77,6 +76,8 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		s.NotifyService.Submit(finalComment)
 	}
 
+	log.Printf("[DEBUG] created commend %+v", finalComment)
+
 	render.Status(r, http.StatusCreated)
 	render.JSON(w, r, &finalComment)
 }

backend/app/store/service/service.go

@@ -1,6 +1,7 @@
 package service
 
 import (
+	"log"
 	"sort"
 	"sync"
 	"time"
@@ -21,7 +22,7 @@ type DataStore struct {
 	AdminStore     admin.Store
 	MaxCommentSize int
 	MaxVotes       int
-	TitleExtractor TitleExtractor
+	TitleExtractor *TitleExtractor
 
 	// granular locks
 	scopedLocks struct {
@@ -59,8 +60,12 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		return "", errors.Wrap(err, "failed to prepare comment")
 	}
 
-	if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil {
-		comment.PostTitle = title
+	if s.TitleExtractor != nil {
+		if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil {
+			comment.PostTitle = title
+		} else {
+			log.Printf("[WARN] failed to set title, %v", err)
+		}
 	}
 	return s.Interface.Create(comment)
 }

backend/app/store/service/service_test.go

@@ -3,6 +3,7 @@ package service
 import (
 	"fmt"
 	"math/rand"
+	"net/http"
 	"os"
 	"strings"
 	"sync"
@@ -68,9 +69,32 @@ func TestService_CreateFromPartial(t *testing.T) {
 	assert.Equal(t, "user", res.User.ID)
 	assert.Equal(t, "name", res.User.Name)
 	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
+	assert.Equal(t, "", res.PostTitle)
 	assert.Equal(t, comment.Votes, res.Votes)
 }
 
+func TestService_CreateFromPartialWithTitle(t *testing.T) {
+	defer os.Remove(testDb)
+	ks := admin.NewStaticKeyStore("secret 123")
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+	comment := store.Comment{
+		Text:      "text",
+		Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
+		Votes:     map[string]bool{"u1": true, "u2": false},
+		User:      store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
+		Locator:   store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"},
+	}
+	id, err := b.Create(comment)
+	assert.NoError(t, err)
+	assert.True(t, id != "", id)
+
+	res, err := b.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)
+	assert.NoError(t, err)
+	t.Logf("%+v", res)
+	assert.Equal(t, "Радио-Т 630 - Радио-Т Подкаст", res.PostTitle)
+}
+
 func TestService_Vote(t *testing.T) {
 	defer os.Remove(testDb)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}

backend/app/store/service/title.go

@@ -38,7 +38,7 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
-		defer resp.Body.Close()
+		defer resp.Body.Close() //nolint
 		if resp.StatusCode != 200 {
 			return nil, errors.Errorf("can't load page %s, code %d", url, resp.StatusCode)
 		}