diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 220982ef..05dd5377 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -586,6 +586,7 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { func (s *ServerCommand) makePicturesStore() (*image.Service, error) { imageServiceParams := image.ServiceParams{ ImageAPI: s.RemarkURL + "/api/v1/picture/", + ProxyAPI: s.RemarkURL + "/api/v1/img", EditDuration: s.EditDuration, MaxSize: s.Image.MaxSize, MaxHeight: s.Image.ResizeHeight, diff --git a/backend/app/rest/proxy/image.go b/backend/app/rest/proxy/image.go index 8284bda9..49cb94d2 100644 --- a/backend/app/rest/proxy/image.go +++ b/backend/app/rest/proxy/image.go @@ -131,7 +131,6 @@ func (p Image) cacheImage(r io.Reader, imgID string) { if err != nil { log.Printf("[WARN] unable to save image to the storage: %+v", err) } - p.ImageService.Submit(func() []string { return []string{imgID} }) } // download an image. diff --git a/backend/app/rest/proxy/image_test.go b/backend/app/rest/proxy/image_test.go index 45c0a309..d158cc7b 100644 --- a/backend/app/rest/proxy/image_test.go +++ b/backend/app/rest/proxy/image_test.go @@ -180,7 +180,6 @@ func TestImage_RoutesCachingImage(t *testing.T) { imageStore.On("Load", mock.Anything).Once().Return(nil, nil) imageStore.On("Save", mock.Anything, mock.Anything).Once().Return(nil) - imageStore.On("Commit", mock.Anything).Once().Return(nil) resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL) require.Nil(t, err) @@ -191,7 +190,6 @@ func TestImage_RoutesCachingImage(t *testing.T) { imageStore.AssertCalled(t, "Load", mock.Anything) imageStore.AssertCalled(t, "Save", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), gopherPNGBytes()) - imageStore.AssertCalled(t, "Commit", mock.Anything) } func TestImage_RoutesUsingCachedImage(t *testing.T) { diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index 7289b506..c94f9623 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "crypto/sha1" //nolint:gosec // not used for cryptography + "encoding/base64" "fmt" "image" // support gif and jpeg images decoding @@ -50,6 +51,7 @@ type Service struct { type ServiceParams struct { EditDuration time.Duration // edit period for comments ImageAPI string // image api matching path + ProxyAPI string // proxy api matching path MaxSize int MaxHeight int MaxWidth int @@ -146,6 +148,21 @@ func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) ids = append(ids, id) } } + if strings.Contains(im, s.ProxyAPI) { + proxiedURL, err := url.Parse(im) + if err != nil { + return + } + imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src")) + if err != nil { + return + } + imgID, err := CachedImgID(string(imgURL)) + if err != nil { + return + } + ids = append(ids, imgID) + } } }) diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index 259cc8c8..19863807 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -3,6 +3,8 @@ package image import ( "bytes" "context" + "encoding/base64" + "fmt" "image" "io" "io/ioutil" @@ -72,24 +74,52 @@ func TestService_WrongFormat(t *testing.T) { } func TestService_ExtractPictures(t *testing.T) { - svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}} - html := `blah foo + svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/", ProxyAPI: "/non_existent"}} + html := `blah foo xyz

123

` ids, err := svc.ExtractPictures(html) require.NoError(t, err) require.Equal(t, 2, len(ids), "two images") assert.Equal(t, "user1/pic1.png", ids[0]) assert.Equal(t, "user2/pic3.png", ids[1]) -} -func TestService_ExtractPictures2(t *testing.T) { - svc := Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"}} - html := "

TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде." + - "

\n\n

Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:

\n\n

\"bjtr0-201906-08110846-i324c.png\"/

\n\n

По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. github.com/go-pkgz/auth v0.5.2.

\n" - ids, err := svc.ExtractPictures(html) + svc = Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/", ProxyAPI: "https://remark42.radio-t.com/api/v1/img"}} + html = `

TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде. +

\n\n

Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). +Например одна из моих вот так выглядит на libraries.io:

\n\n +

bjtr0-201906-08110846-i324c.png

\n\n

+По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. +github.com/go-pkgz/auth v0.5.2.

\n` + ids, err = svc.ExtractPictures(html) require.NoError(t, err) require.Equal(t, 1, len(ids), "one image in") assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0]) + + // proxied image + html = `cat.png` + ids, err = svc.ExtractPictures(html) + require.NoError(t, err) + require.Equal(t, 1, len(ids), "one image in") + assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0]) + + // bad url + html = `` + ids, err = svc.ExtractPictures(html) + require.NoError(t, err) + require.Empty(t, ids) + + // bad src + html = `` + ids, err = svc.ExtractPictures(html) + require.NoError(t, err) + require.Empty(t, ids) + + // good src with bad content + badURL := base64.URLEncoding.EncodeToString([]byte(" http://foo.bar")) + html = fmt.Sprintf(``, badURL) + ids, err = svc.ExtractPictures(html) + require.NoError(t, err) + require.Empty(t, ids) } func TestService_Cleanup(t *testing.T) { diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index 1a494bb9..a7748cd0 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -19,7 +19,6 @@ import ( "github.com/go-pkgz/lgr" "github.com/pkg/errors" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" bolt "go.etcd.io/bbolt" @@ -100,7 +99,6 @@ func TestService_CreateFromPartial(t *testing.T) { } func TestService_CreateFromPartialWithTitle(t *testing.T) { - ks := admin.NewStaticKeyStore("secret 123") eng, teardown := prepStoreEngine(t) defer teardown() @@ -1279,8 +1277,14 @@ func TestService_submitImages(t *testing.T) { lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc) mockStore := image.MockStore{} - mockStore.On("Commit", mock.Anything).Times(2).Return(nil) - imgSvc := image.NewService(&mockStore, image.ServiceParams{EditDuration: 50 * time.Millisecond}) + mockStore.On("Commit", "dev/pic1.png").Once().Return(nil) + mockStore.On("Commit", "dev/pic2.png").Once().Return(nil) + imgSvc := image.NewService(&mockStore, + image.ServiceParams{ + EditDuration: 50 * time.Millisecond, + ImageAPI: "/", + ProxyAPI: "/non_existent", + }) defer imgSvc.Close(context.TODO()) // two comments for https://radio-t.com @@ -1301,6 +1305,7 @@ func TestService_submitImages(t *testing.T) { b.submitImages(c.Locator, c.ID) time.Sleep(250 * time.Millisecond) + mockStore.AssertNumberOfCalls(t, "Commit", 2) } func TestService_ResubmitStagingImages(t *testing.T) { @@ -1309,6 +1314,7 @@ func TestService_ResubmitStagingImages(t *testing.T) { image.ServiceParams{ EditDuration: 10 * time.Millisecond, ImageAPI: "http://127.0.0.1:8080/api/v1/picture/", + ProxyAPI: "http://127.0.0.1:8080/api/v1/img", }) defer imgSvc.Close(context.TODO()) @@ -1321,6 +1327,7 @@ func TestService_ResubmitStagingImages(t *testing.T) { ID: "id-0", Text: `startrails_01.jpg
cat.png
+ cat.png
boat.png`, Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, @@ -1337,10 +1344,11 @@ func TestService_ResubmitStagingImages(t *testing.T) { // wait for Submit goroutine to commit image mockStore.On("Commit", "dev_user/bqf122eq9r8ad657n3ng").Once().Return(nil) mockStore.On("Commit", "dev_user/bqf321eq9r8ad657n3ng").Once().Return(nil) + mockStore.On("Commit", "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e").Once().Return(nil) time.Sleep(time.Millisecond * 100) mockStore.AssertNumberOfCalls(t, "Info", 1) - mockStore.AssertNumberOfCalls(t, "Commit", 2) + mockStore.AssertNumberOfCalls(t, "Commit", 3) // empty answer mockStoreEmpty := image.MockStore{}