mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

commit proxy images same way as normally uploaded ones

Browse Source
This commit is contained in:
Dmitry Verkhoturov
2020-04-25 12:15:26 -05:00
committed by Umputun
parent 5579bd1a88
commit 71dcb13a71
6 changed files with 69 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/cmd/server.go
+1
View File
@@ -586,6 +586,7 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
imageServiceParams := image.ServiceParams{
ImageAPI: s.RemarkURL + "/api/v1/picture/",
ProxyAPI: s.RemarkURL + "/api/v1/img",
EditDuration: s.EditDuration,
MaxSize: s.Image.MaxSize,
MaxHeight: s.Image.ResizeHeight,
backend/app/rest/proxy/image.go
-1
View File
@@ -131,7 +131,6 @@ func (p Image) cacheImage(r io.Reader, imgID string) {
if err != nil {
log.Printf("[WARN] unable to save image to the storage: %+v", err)
}
p.ImageService.Submit(func() []string { return []string{imgID} })
}
// download an image.
backend/app/rest/proxy/image_test.go
-2
View File
@@ -180,7 +180,6 @@ func TestImage_RoutesCachingImage(t *testing.T) {
imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
imageStore.On("Save", mock.Anything, mock.Anything).Once().Return(nil)
imageStore.On("Commit", mock.Anything).Once().Return(nil)
resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
require.Nil(t, err)
@@ -191,7 +190,6 @@ func TestImage_RoutesCachingImage(t *testing.T) {
imageStore.AssertCalled(t, "Load", mock.Anything)
imageStore.AssertCalled(t, "Save", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), gopherPNGBytes())
imageStore.AssertCalled(t, "Commit", mock.Anything)
}
func TestImage_RoutesUsingCachedImage(t *testing.T) {
backend/app/store/image/image.go
+17
View File
@@ -9,6 +9,7 @@ import (
"bytes"
"context"
"crypto/sha1" //nolint:gosec // not used for cryptography
"encoding/base64"
"fmt"
"image"
// support gif and jpeg images decoding
@@ -50,6 +51,7 @@ type Service struct {
type ServiceParams struct {
EditDuration time.Duration // edit period for comments
ImageAPI string // image api matching path
ProxyAPI string // proxy api matching path
MaxSize int
MaxHeight int
MaxWidth int
@@ -146,6 +148,21 @@ func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error)
ids = append(ids, id)
}
}
if strings.Contains(im, s.ProxyAPI) {
proxiedURL, err := url.Parse(im)
if err != nil {
return
}
imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
if err != nil {
return
}
imgID, err := CachedImgID(string(imgURL))
if err != nil {
return
}
ids = append(ids, imgID)
}
}
})
backend/app/store/image/image_test.go
+38 -8
View File
@@ -3,6 +3,8 @@ package image
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"image"
"io"
"io/ioutil"
@@ -72,24 +74,52 @@ func TestService_WrongFormat(t *testing.T) {
}
func TestService_ExtractPictures(t *testing.T) {
svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}}
html := `blah <img src="/blah/user1/pic1.png"/> foo
svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/", ProxyAPI: "/non_existent"}}
html := `blah <img src="/blah/user1/pic1.png"/> foo
<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/> <img src="https://i.ibb.co/0cqqqnD/ezgif-5-3b07b6b97610.png" alt="">`
ids, err := svc.ExtractPictures(html)
require.NoError(t, err)
require.Equal(t, 2, len(ids), "two images")
assert.Equal(t, "user1/pic1.png", ids[0])
assert.Equal(t, "user2/pic3.png", ids[1])
}
func TestService_ExtractPictures2(t *testing.T) {
svc := Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"}}
html := "<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде." +
"</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:</p>\n\n<p><img src=\"https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png\" alt=\"bjtr0-201906-08110846-i324c.png\"/></p>\n\n<p>По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. <code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n"
ids, err := svc.ExtractPictures(html)
svc = Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/", ProxyAPI: "https://remark42.radio-t.com/api/v1/img"}}
html = `<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде.
</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля).
Например одна из моих вот так выглядит на libraries.io:</p>\n\n
<p><img src="https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png" alt="bjtr0-201906-08110846-i324c.png"/></p>\n\n<p>
По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е.
<code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n`
ids, err = svc.ExtractPictures(html)
require.NoError(t, err)
require.Equal(t, 1, len(ids), "one image in")
assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0])
// proxied image
html = `<img src="https://remark42.radio-t.com/api/v1/img?src=aHR0cHM6Ly9ob21lcGFnZXMuY2FlLndpc2MuZWR1L35lY2U1MzMvaW1hZ2VzL2JvYXQucG5n" alt="cat.png">`
ids, err = svc.ExtractPictures(html)
require.NoError(t, err)
require.Equal(t, 1, len(ids), "one image in")
assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0])
// bad url
html = `<img src=" https://remark42.radio-t.com/api/v1/img">`
ids, err = svc.ExtractPictures(html)
require.NoError(t, err)
require.Empty(t, ids)
// bad src
html = `<img src="https://remark42.radio-t.com/api/v1/img?src=bad">`
ids, err = svc.ExtractPictures(html)
require.NoError(t, err)
require.Empty(t, ids)
// good src with bad content
badURL := base64.URLEncoding.EncodeToString([]byte(" http://foo.bar"))
html = fmt.Sprintf(`<img src="https://remark42.radio-t.com/api/v1/img?src=%s">`, badURL)
ids, err = svc.ExtractPictures(html)
require.NoError(t, err)
require.Empty(t, ids)
}
func TestService_Cleanup(t *testing.T) {
backend/app/store/service/service_test.go
+13 -5
View File
@@ -19,7 +19,6 @@ import (
"github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
@@ -100,7 +99,6 @@ func TestService_CreateFromPartial(t *testing.T) {
}
func TestService_CreateFromPartialWithTitle(t *testing.T) {
ks := admin.NewStaticKeyStore("secret 123")
eng, teardown := prepStoreEngine(t)
defer teardown()
@@ -1279,8 +1277,14 @@ func TestService_submitImages(t *testing.T) {
lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
mockStore := image.MockStore{}
mockStore.On("Commit", mock.Anything).Times(2).Return(nil)
imgSvc := image.NewService(&mockStore, image.ServiceParams{EditDuration: 50 * time.Millisecond})
mockStore.On("Commit", "dev/pic1.png").Once().Return(nil)
mockStore.On("Commit", "dev/pic2.png").Once().Return(nil)
imgSvc := image.NewService(&mockStore,
image.ServiceParams{
EditDuration: 50 * time.Millisecond,
ImageAPI: "/",
ProxyAPI: "/non_existent",
})
defer imgSvc.Close(context.TODO())
// two comments for https://radio-t.com
@@ -1301,6 +1305,7 @@ func TestService_submitImages(t *testing.T) {
b.submitImages(c.Locator, c.ID)
time.Sleep(250 * time.Millisecond)
mockStore.AssertNumberOfCalls(t, "Commit", 2)
}
func TestService_ResubmitStagingImages(t *testing.T) {
@@ -1309,6 +1314,7 @@ func TestService_ResubmitStagingImages(t *testing.T) {
image.ServiceParams{
EditDuration: 10 * time.Millisecond,
ImageAPI: "http://127.0.0.1:8080/api/v1/picture/",
ProxyAPI: "http://127.0.0.1:8080/api/v1/img",
})
defer imgSvc.Close(context.TODO())
@@ -1321,6 +1327,7 @@ func TestService_ResubmitStagingImages(t *testing.T) {
ID: "id-0",
Text: `<img src="http://127.0.0.1:8080/api/v1/picture/dev_user/bqf122eq9r8ad657n3ng" alt="startrails_01.jpg"><br/>
<img src="http://127.0.0.1:8080/api/v1/picture/dev_user/bqf321eq9r8ad657n3ng" alt="cat.png"><br/>
<img src="http://127.0.0.1:8080/api/v1/img?src=aHR0cHM6Ly9ob21lcGFnZXMuY2FlLndpc2MuZWR1L35lY2U1MzMvaW1hZ2VzL2JvYXQucG5n" alt="cat.png"><br/>
<img src="https://homepages.cae.wisc.edu/~ece533/images/boat.png" alt="boat.png">`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
@@ -1337,10 +1344,11 @@ func TestService_ResubmitStagingImages(t *testing.T) {
// wait for Submit goroutine to commit image
mockStore.On("Commit", "dev_user/bqf122eq9r8ad657n3ng").Once().Return(nil)
mockStore.On("Commit", "dev_user/bqf321eq9r8ad657n3ng").Once().Return(nil)
mockStore.On("Commit", "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e").Once().Return(nil)
time.Sleep(time.Millisecond * 100)
mockStore.AssertNumberOfCalls(t, "Info", 1)
mockStore.AssertNumberOfCalls(t, "Commit", 2)
mockStore.AssertNumberOfCalls(t, "Commit", 3)
// empty answer
mockStoreEmpty := image.MockStore{}