diff --git a/backend/app/store/service/title.go b/backend/app/store/service/title.go index 30c1e021..a8a72c80 100644 --- a/backend/app/store/service/title.go +++ b/backend/app/store/service/title.go @@ -34,9 +34,9 @@ func NewTitleExtractor(client http.Client) *TitleExtractor { // Get page for url and return title func (t *TitleExtractor) Get(url string) (string, error) { - + client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport} b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) { - resp, err := t.client.Get(url) + resp, err := client.Get(url) if err != nil { return nil, errors.Wrapf(err, "failed to load page %s", url) } diff --git a/backend/app/store/service/title_test.go b/backend/app/store/service/title_test.go index a5a35c74..c0a2d94f 100644 --- a/backend/app/store/service/title_test.go +++ b/backend/app/store/service/title_test.go @@ -4,11 +4,13 @@ import ( "fmt" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "testing" "time" + "github.com/go-pkgz/syncs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -61,3 +63,33 @@ func TestTitle_Get(t *testing.T) { } assert.Equal(t, int32(1), atomic.LoadInt32(&hits)) } + +func TestTitle_GetConcurrent(t *testing.T) { + body := "" + for n := 0; n < 1000; n++ { + body += "something something blah blah\n" + } + ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}) + var hits int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.String(), "/good") { + atomic.AddInt32(&hits, 1) + w.Write([]byte(fmt.Sprintf("blah 123 %s%s", r.URL.String(), body))) + return + } + w.WriteHeader(404) + })) + + g := syncs.NewSizedGroup(10) + + for i := 0; i < 100; i++ { + i := i + g.Go(func() { + title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(i)) + require.Nil(t, err) + assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(i), title) + }) + } + g.Wait() + assert.Equal(t, int32(100), atomic.LoadInt32(&hits)) +}