fix shared title client caused races

2019-02-02 17:50:13 -06:00
parent 0fe3f6dd7d
commit a731856bb1
2 changed files with 34 additions and 2 deletions
@@ -34,9 +34,9 @@ func NewTitleExtractor(client http.Client) *TitleExtractor {

 // Get page for url and return title
 func (t *TitleExtractor) Get(url string) (string, error) {
-
+	client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
 	b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) {
-		resp, err := t.client.Get(url)
+		resp, err := client.Get(url)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
 	"time"

+	"github.com/go-pkgz/syncs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -61,3 +63,33 @@ func TestTitle_Get(t *testing.T) {
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
 }
+
+func TestTitle_GetConcurrent(t *testing.T) {
+	body := ""
+	for n := 0; n < 1000; n++ {
+		body += "something something blah blah\n"
+	}
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.String(), "/good") {
+			atomic.AddInt32(&hits, 1)
+			w.Write([]byte(fmt.Sprintf("<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body)))
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	g := syncs.NewSizedGroup(10)
+
+	for i := 0; i < 100; i++ {
+		i := i
+		g.Go(func() {
+			title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(i))
+			require.Nil(t, err)
+			assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(i), title)
+		})
+	}
+	g.Wait()
+	assert.Equal(t, int32(100), atomic.LoadInt32(&hits))
+}