diff --git a/backend/app/main.go b/backend/app/main.go
index 483ed189..acc7a748 100644
--- a/backend/app/main.go
+++ b/backend/app/main.go
@@ -16,6 +16,7 @@ import (
"github.com/hashicorp/logutils"
"github.com/jessevdk/go-flags"
"github.com/pkg/errors"
+ "github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/migrator"
@@ -217,17 +218,20 @@ func New(opts Opts) (*Application, error) {
}
authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts)
+ imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}
+ commentFormater := store.NewCommentFormater(imgProxy)
srv := &api.Rest{
- Version: revision,
- DataService: dataService,
- Exporter: exporter,
- WebRoot: opts.WebRoot,
- RemarkURL: opts.RemarkURL,
- ImageProxy: &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL},
- AvatarProxy: avatarProxy,
- ReadOnlyAge: opts.ReadOnlyAge,
- SharedSecret: opts.SharedSecret,
+ Version: revision,
+ DataService: dataService,
+ Exporter: exporter,
+ WebRoot: opts.WebRoot,
+ RemarkURL: opts.RemarkURL,
+ ImageProxy: imgProxy,
+ CommentFormater: commentFormater,
+ AvatarProxy: avatarProxy,
+ ReadOnlyAge: opts.ReadOnlyAge,
+ SharedSecret: opts.SharedSecret,
Authenticator: auth.Authenticator{
JWTService: jwtService,
AdminEmail: opts.AdminEmail,
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index c83db44c..7746e936 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -20,7 +20,6 @@ import (
"github.com/go-chi/cors"
"github.com/go-chi/render"
"github.com/pkg/errors"
- "gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest"
@@ -33,13 +32,16 @@ import (
// Rest is a rest access server
type Rest struct {
- Version string
+ Version string
+
DataService *service.DataStore
Authenticator auth.Authenticator
Exporter migrator.Exporter
Cache cache.LoadingCache
AvatarProxy *proxy.Avatar
ImageProxy *proxy.Image
+ CommentFormater *store.CommentFormater
+
WebRoot string
RemarkURL string
ReadOnlyAge int
@@ -57,10 +59,6 @@ type Rest struct {
const hardBodyLimit = 1024 * 64 // limit size of body
-var mdExt = blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
- blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
- blackfriday.BackslashLineBreak | blackfriday.Autolink
-
type commentsWithInfo struct {
Comments []store.Comment `json:"comments"`
Info store.PostInfo `json:"info,omitempty"`
diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go
index 62a2163a..b8250593 100644
--- a/backend/app/rest/api/rest_private.go
+++ b/backend/app/rest/api/rest_private.go
@@ -14,7 +14,6 @@ import (
"github.com/go-chi/chi"
"github.com/go-chi/render"
"github.com/hashicorp/go-multierror"
- "gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/auth"
@@ -44,8 +43,8 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
return
}
- comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
- comment.Text = s.ImageProxy.Convert(comment.Text)
+ comment = s.CommentFormater.Format(comment)
+
// check if user blocked
if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
@@ -109,10 +108,8 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
- text := string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))) // render markdown
- text = s.ImageProxy.Convert(text)
editReq := service.EditRequest{
- Text: text,
+ Text: s.CommentFormater.FormatText(edit.Text),
Orig: edit.Text,
Summary: edit.Summary,
}
diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go
index bf397ab0..5315ebc0 100644
--- a/backend/app/rest/api/rest_public.go
+++ b/backend/app/rest/api/rest_public.go
@@ -10,7 +10,6 @@ import (
"github.com/go-chi/chi"
"github.com/go-chi/render"
- "gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/cache"
@@ -79,10 +78,7 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
- //comment.Text = string(blackfriday.Run([]byte(comment.Text),
- // blackfriday.WithRenderer(bfchroma.NewRenderer(bfchroma.WithoutAutodetect()))))
- comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
- comment.Text = s.ImageProxy.Convert(comment.Text)
+ comment = s.CommentFormater.Format(comment)
comment.Sanitize()
render.HTML(w, r, comment.Text)
}
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index 9a691c76..218dfb05 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -91,13 +91,14 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
AdminEmail: "admin@remark-42.com",
JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour),
},
- Exporter: &migrator.Remark{DataStore: dataStore},
- Cache: &cache.Nop{},
- WebRoot: "/tmp",
- RemarkURL: "https://demo.remark42.com",
- AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"},
- ImageProxy: &proxy.Image{},
- ReadOnlyAge: 10,
+ Exporter: &migrator.Remark{DataStore: dataStore},
+ Cache: &cache.Nop{},
+ WebRoot: "/tmp",
+ RemarkURL: "https://demo.remark42.com",
+ AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"},
+ ImageProxy: &proxy.Image{},
+ ReadOnlyAge: 10,
+ CommentFormater: store.NewCommentFormater(&proxy.Image{}),
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
diff --git a/backend/app/store/comment.go b/backend/app/store/comment.go
index 0fe916bb..51c5fb38 100644
--- a/backend/app/store/comment.go
+++ b/backend/app/store/comment.go
@@ -2,12 +2,9 @@ package store
import (
"html/template"
- "net/url"
"regexp"
- "strings"
"time"
- "github.com/PuerkitoBio/goquery"
"github.com/microcosm-cc/bluemonday"
)
@@ -102,43 +99,8 @@ func (c *Comment) Sanitize() {
p := bluemonday.UGCPolicy()
p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
c.Text = p.Sanitize(c.Text)
- c.Text = shortenAutoLinks(c.Text, shortURLLen)
c.Orig = p.Sanitize(c.Orig)
c.User.ID = template.HTMLEscapeString(c.User.ID)
c.User.Name = template.HTMLEscapeString(c.User.Name)
c.User.Picture = p.Sanitize(c.User.Picture)
}
-
-// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
-func shortenAutoLinks(commentHTML string, max int) (resHTML string) {
- doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
- if err != nil {
- return commentHTML
- }
- doc.Find("a").Each(func(i int, s *goquery.Selection) {
- if href, ok := s.Attr("href"); ok {
- if href != s.Text() || len(href) < max+3 || max < 3 {
- return
- }
- url, e := url.Parse(href)
- if e != nil {
- return
- }
- url.Path, url.RawQuery, url.Fragment = "", "", ""
- host := url.String()
- if host == "" {
- return
- }
- short := href[:max-3]
- if len(short) < len(host) {
- short = host
- }
- s.SetText(short + "...")
- }
- })
- resHTML, err = doc.Find("body").Html()
- if err != nil {
- return commentHTML
- }
- return resHTML
-}
diff --git a/backend/app/store/comment_test.go b/backend/app/store/comment_test.go
index 1ce03754..2829382b 100644
--- a/backend/app/store/comment_test.go
+++ b/backend/app/store/comment_test.go
@@ -17,21 +17,21 @@ func TestComment_Sanitize(t *testing.T) {
{
inp: Comment{
Text: `blah XSS` + "\n\t",
- User: User{ID: `username`},
+ User: User{ID: `username`, Name: "name "},
},
out: Comment{
Text: "blah XSS\n\t",
- User: User{ID: `<a href="http://blah.com">username</a>`},
+ User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name <b/>"},
},
},
{
inp: Comment{
- Text: `blah https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/` + "\n\t",
- User: User{ID: `username`},
+ Text: "blah 123" + "\n\t",
+ User: User{ID: "id", Name: "xyz"},
},
out: Comment{
- Text: `blah https://www.reddit.com/r/golang/comments/8jdo...` + "\n\t",
- User: User{ID: `<a href="http://blah.com">username</a>`},
+ Text: `blah 123` + "\n\t",
+ User: User{ID: "id", Name: "xyz"},
},
},
}
@@ -120,46 +120,3 @@ func TestComment_SetDeletedHard(t *testing.T) {
assert.False(t, comment.Pin)
assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
}
-
-func TestComment_ShortenAutoLinks(t *testing.T) {
- tbl := []struct {
- max int
- in, out string
- }{
- {32, "", ""},
- {32, "text", "text"},
- {32, "asd
", "asd
"},
- {5, `incorrect-url`, `incorrect-url`},
- {32, `some text, not href`, `some text, not href`},
- {
- 32,
- `https://blah.com/a/b/c/d?g=123#anc`,
- `https://blah.com/a/b/c/d?g=123#anc`,
- },
- {
- 31,
- `https://blah.com/a/b/c/d?g=123#anc`,
- `https://blah.com/a/b/c/d?g=1...`,
- },
- {
- 15,
- `https://blah.com/a/b/c/d?g=123#anc`,
- `https://blah.com...`,
- },
- {
- 3,
- `https://blah.com/a/b/c/d?g=123#anc`,
- `https://blah.com...`,
- },
- {
- -1,
- `https://blah.com/a/b/c/d?g=123#anc`,
- `https://blah.com/a/b/c/d?g=123#anc`,
- },
- }
-
- for n, tt := range tbl {
- got := shortenAutoLinks(tt.in, tt.max)
- assert.Equalf(t, tt.out, got, "check #%d", n)
- }
-}
diff --git a/backend/app/store/fomrater.go b/backend/app/store/fomrater.go
new file mode 100644
index 00000000..2154d4f3
--- /dev/null
+++ b/backend/app/store/fomrater.go
@@ -0,0 +1,79 @@
+package store
+
+import (
+ "net/url"
+ "strings"
+
+ "github.com/PuerkitoBio/goquery"
+ blackfriday "gopkg.in/russross/blackfriday.v2"
+)
+
+// CommentFormater implements all generic formatings ops on comment
+type CommentFormater struct {
+ converters []CommentConverter
+}
+
+// CommentConverter defines interface to convert some parts of commentHTML
+// Passed at creation time and does client-defined convertions, like image proxy link change
+type CommentConverter interface {
+ Convert(text string) string
+}
+
+// NewCommentFormater makes CommentFormater
+func NewCommentFormater(converters ...CommentConverter) *CommentFormater {
+ return &CommentFormater{converters: converters}
+}
+
+// Format comment fields
+func (f *CommentFormater) Format(c Comment) Comment {
+ c.Text = f.FormatText(c.Text)
+ return c
+}
+
+// FormatText formatting line
+func (f *CommentFormater) FormatText(txt string) (res string) {
+ mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
+ blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
+ blackfriday.BackslashLineBreak | blackfriday.Autolink
+ res = string(blackfriday.Run([]byte(txt), blackfriday.WithExtensions(mdExt)))
+ for _, conv := range f.converters {
+ res = conv.Convert(res)
+
+ }
+ res = f.shortenAutoLinks(res, shortURLLen)
+ return res
+}
+
+// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
+func (f *CommentFormater) shortenAutoLinks(commentHTML string, max int) (resHTML string) {
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+ if err != nil {
+ return commentHTML
+ }
+ doc.Find("a").Each(func(i int, s *goquery.Selection) {
+ if href, ok := s.Attr("href"); ok {
+ if href != s.Text() || len(href) < max+3 || max < 3 {
+ return
+ }
+ url, e := url.Parse(href)
+ if e != nil {
+ return
+ }
+ url.Path, url.RawQuery, url.Fragment = "", "", ""
+ host := url.String()
+ if host == "" {
+ return
+ }
+ short := href[:max-3]
+ if len(short) < len(host) {
+ short = host
+ }
+ s.SetText(short + "...")
+ }
+ })
+ resHTML, err = doc.Find("body").Html()
+ if err != nil {
+ return commentHTML
+ }
+ return resHTML
+}
diff --git a/backend/app/store/formater_test.go b/backend/app/store/formater_test.go
new file mode 100644
index 00000000..cd33f24a
--- /dev/null
+++ b/backend/app/store/formater_test.go
@@ -0,0 +1,98 @@
+package store
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+type mockConvertor struct{}
+
+func (m mockConvertor) Convert(text string) string { return text + "!converted" }
+
+func TestFormater_FormatText(t *testing.T) {
+ tbl := []struct {
+ in, out string
+ }{
+ {"", "!converted"},
+ {"12345 abc", "12345 abc
\n!converted"},
+ {"**xyz** _aaa_", "xyz aaa
\n!converted"},
+ {
+ "http://127.0.0.1/some-long-link/12345/678901234567890", "http://127.0.0.1/some-long-link/12345/6789012...
\n!converted",
+ },
+ }
+ f := NewCommentFormater(mockConvertor{})
+ for n, tt := range tbl {
+ assert.Equal(t, tt.out, f.FormatText(tt.in), "check #%d", n)
+ }
+}
+
+func TestFormater_FormatTextNoConvertor(t *testing.T) {
+ f := NewCommentFormater()
+ assert.Equal(t, "12345
\n", f.FormatText("12345"))
+}
+
+func TestFormater_FormatComment(t *testing.T) {
+ comment := Comment{
+ Text: `blah`,
+ User: User{ID: "username"},
+ ParentID: "p123",
+ ID: "123",
+ Locator: Locator{SiteID: "site", URL: "url"},
+ Score: 10,
+ Pin: true,
+ Deleted: true,
+ Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
+ Votes: map[string]bool{"uu": true},
+ }
+
+ f := NewCommentFormater(mockConvertor{})
+ exp := comment
+ exp.Text = "blah
\n!converted"
+ assert.Equal(t, exp, f.Format(comment))
+}
+
+func TestFormater_ShortenAutoLinks(t *testing.T) {
+ f := NewCommentFormater(nil)
+ tbl := []struct {
+ max int
+ in, out string
+ }{
+ {32, "", ""},
+ {32, "text", "text"},
+ {32, "asd
", "asd
"},
+ {5, `incorrect-url`, `incorrect-url`},
+ {32, `some text, not href`, `some text, not href`},
+ {
+ 32,
+ `https://blah.com/a/b/c/d?g=123#anc`,
+ `https://blah.com/a/b/c/d?g=123#anc`,
+ },
+ {
+ 31,
+ `https://blah.com/a/b/c/d?g=123#anc`,
+ `https://blah.com/a/b/c/d?g=1...`,
+ },
+ {
+ 15,
+ `https://blah.com/a/b/c/d?g=123#anc`,
+ `https://blah.com...`,
+ },
+ {
+ 3,
+ `https://blah.com/a/b/c/d?g=123#anc`,
+ `https://blah.com...`,
+ },
+ {
+ -1,
+ `https://blah.com/a/b/c/d?g=123#anc`,
+ `https://blah.com/a/b/c/d?g=123#anc`,
+ },
+ }
+
+ for n, tt := range tbl {
+ got := f.shortenAutoLinks(tt.in, tt.max)
+ assert.Equalf(t, tt.out, got, "check #%d", n)
+ }
+}
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index d26d898b..98f05fcf 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -43,7 +43,6 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
if comment.Votes == nil {
comment.Votes = make(map[string]bool)
}
-
comment.Sanitize() // clear potentially dangerous js from all parts of comment
secret, err := s.KeyStore.Get(comment.Locator.SiteID)