mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature/comment-formater (#186)

Browse Source 
* extract comment formatter functuionality to allow md in #156

* add abilty to pass multiple convertors to formater
This commit is contained in:
Umputun
2018-08-10 19:35:45 -05:00
committed by GitHub
parent f15af0c7dc
commit 2ff955edf1
10 changed files with 212 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/main.go
+13 -9
View File
@@ -16,6 +16,7 @@ import (
"github.com/hashicorp/logutils"
"github.com/jessevdk/go-flags"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/migrator"
@@ -217,17 +218,20 @@ func New(opts Opts) (*Application, error) {
}
authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts)
imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}
commentFormater := store.NewCommentFormater(imgProxy)
srv := &api.Rest{
Version: revision,
DataService: dataService,
Exporter: exporter,
WebRoot: opts.WebRoot,
RemarkURL: opts.RemarkURL,
ImageProxy: &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL},
AvatarProxy: avatarProxy,
ReadOnlyAge: opts.ReadOnlyAge,
SharedSecret: opts.SharedSecret,
Version: revision,
DataService: dataService,
Exporter: exporter,
WebRoot: opts.WebRoot,
RemarkURL: opts.RemarkURL,
ImageProxy: imgProxy,
CommentFormater: commentFormater,
AvatarProxy: avatarProxy,
ReadOnlyAge: opts.ReadOnlyAge,
SharedSecret: opts.SharedSecret,
Authenticator: auth.Authenticator{
JWTService: jwtService,
AdminEmail: opts.AdminEmail,
backend/app/rest/api/rest.go
+4 -6
View File
@@ -20,7 +20,6 @@ import (
"github.com/go-chi/cors"
"github.com/go-chi/render"
"github.com/pkg/errors"
"gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest"
@@ -33,13 +32,16 @@ import (
// Rest is a rest access server
type Rest struct {
Version string
Version string
DataService *service.DataStore
Authenticator auth.Authenticator
Exporter migrator.Exporter
Cache cache.LoadingCache
AvatarProxy *proxy.Avatar
ImageProxy *proxy.Image
CommentFormater *store.CommentFormater
WebRoot string
RemarkURL string
ReadOnlyAge int
@@ -57,10 +59,6 @@ type Rest struct {
const hardBodyLimit = 1024 * 64 // limit size of body
var mdExt = blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
blackfriday.BackslashLineBreak | blackfriday.Autolink
type commentsWithInfo struct {
Comments []store.Comment `json:"comments"`
Info store.PostInfo `json:"info,omitempty"`
backend/app/rest/api/rest_private.go
+3 -6
View File
@@ -14,7 +14,6 @@ import (
"github.com/go-chi/chi"
"github.com/go-chi/render"
"github.com/hashicorp/go-multierror"
"gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/auth"
@@ -44,8 +43,8 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
return
}
comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
comment.Text = s.ImageProxy.Convert(comment.Text)
comment = s.CommentFormater.Format(comment)
// check if user blocked
if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
@@ -109,10 +108,8 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
text := string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))) // render markdown
text = s.ImageProxy.Convert(text)
editReq := service.EditRequest{
Text: text,
Text: s.CommentFormater.FormatText(edit.Text),
Orig: edit.Text,
Summary: edit.Summary,
}
backend/app/rest/api/rest_public.go
+1 -5
View File
@@ -10,7 +10,6 @@ import (
"github.com/go-chi/chi"
"github.com/go-chi/render"
"gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/cache"
@@ -79,10 +78,7 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
//comment.Text = string(blackfriday.Run([]byte(comment.Text),
// blackfriday.WithRenderer(bfchroma.NewRenderer(bfchroma.WithoutAutodetect()))))
comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
comment.Text = s.ImageProxy.Convert(comment.Text)
comment = s.CommentFormater.Format(comment)
comment.Sanitize()
render.HTML(w, r, comment.Text)
}
backend/app/rest/api/rest_test.go
+8 -7
View File
@@ -91,13 +91,14 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
AdminEmail: "admin@remark-42.com",
JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour),
},
Exporter: &migrator.Remark{DataStore: dataStore},
Cache: &cache.Nop{},
WebRoot: "/tmp",
RemarkURL: "https://demo.remark42.com",
AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"},
ImageProxy: &proxy.Image{},
ReadOnlyAge: 10,
Exporter: &migrator.Remark{DataStore: dataStore},
Cache: &cache.Nop{},
WebRoot: "/tmp",
RemarkURL: "https://demo.remark42.com",
AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"},
ImageProxy: &proxy.Image{},
ReadOnlyAge: 10,
CommentFormater: store.NewCommentFormater(&proxy.Image{}),
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
backend/app/store/comment.go
-38
View File
@@ -2,12 +2,9 @@ package store
import (
"html/template"
"net/url"
"regexp"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/microcosm-cc/bluemonday"
)
@@ -102,43 +99,8 @@ func (c *Comment) Sanitize() {
p := bluemonday.UGCPolicy()
p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
c.Text = p.Sanitize(c.Text)
c.Text = shortenAutoLinks(c.Text, shortURLLen)
c.Orig = p.Sanitize(c.Orig)
c.User.ID = template.HTMLEscapeString(c.User.ID)
c.User.Name = template.HTMLEscapeString(c.User.Name)
c.User.Picture = p.Sanitize(c.User.Picture)
}
// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
func shortenAutoLinks(commentHTML string, max int) (resHTML string) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
if err != nil {
return commentHTML
}
doc.Find("a").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Attr("href"); ok {
if href != s.Text() || len(href) < max+3 || max < 3 {
return
}
url, e := url.Parse(href)
if e != nil {
return
}
url.Path, url.RawQuery, url.Fragment = "", "", ""
host := url.String()
if host == "" {
return
}
short := href[:max-3]
if len(short) < len(host) {
short = host
}
s.SetText(short + "...")
}
})
resHTML, err = doc.Find("body").Html()
if err != nil {
return commentHTML
}
return resHTML
}
backend/app/store/comment_test.go
+6 -49
View File
@@ -17,21 +17,21 @@ func TestComment_Sanitize(t *testing.T) {
{
inp: Comment{
Text: `blah <a href="javascript:alert('XSS1')" onmouseover="alert('XSS2')">XSS</a>` + "\n\t",
User: User{ID: `<a href="http://blah.com">username</a>`},
User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name <b/>"},
},
out: Comment{
Text: "blah XSS\n\t",
User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`},
User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`, Name: "name &lt;b/&gt;"},
},
},
{
inp: Comment{
Text: `blah <a href="https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/">https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/</a>` + "\n\t",
User: User{ID: `<a href="http://blah.com">username</a>`},
Text: "blah 123" + "\n\t",
User: User{ID: "id", Name: "xyz"},
},
out: Comment{
Text: `blah <a href="https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/" rel="nofollow">https://www.reddit.com/r/golang/comments/8jdo...</a>` + "\n\t",
User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`},
Text: `blah 123` + "\n\t",
User: User{ID: "id", Name: "xyz"},
},
},
}
@@ -120,46 +120,3 @@ func TestComment_SetDeletedHard(t *testing.T) {
assert.False(t, comment.Pin)
assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
}
func TestComment_ShortenAutoLinks(t *testing.T) {
tbl := []struct {
max int
in, out string
}{
{32, "", ""},
{32, "text", "text"},
{32, "<p>asd</p>", "<p>asd</p>"},
{5, `<a href="incorrect-url">incorrect-url</a>`, `<a href="incorrect-url">incorrect-url</a>`},
{32, `<a href="https://blah.com">some text, not href</a>`, `<a href="https://blah.com">some text, not href</a>`},
{
32,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
{
31,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=1...</a>`,
},
{
15,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
3,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
-1,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
}
for n, tt := range tbl {
got := shortenAutoLinks(tt.in, tt.max)
assert.Equalf(t, tt.out, got, "check #%d", n)
}
}
backend/app/store/fomrater.go
+79
View File
@@ -0,0 +1,79 @@
package store
import (
"net/url"
"strings"
"github.com/PuerkitoBio/goquery"
blackfriday "gopkg.in/russross/blackfriday.v2"
)
// CommentFormater implements all generic formatings ops on comment
type CommentFormater struct {
converters []CommentConverter
}
// CommentConverter defines interface to convert some parts of commentHTML
// Passed at creation time and does client-defined convertions, like image proxy link change
type CommentConverter interface {
Convert(text string) string
}
// NewCommentFormater makes CommentFormater
func NewCommentFormater(converters ...CommentConverter) *CommentFormater {
return &CommentFormater{converters: converters}
}
// Format comment fields
func (f *CommentFormater) Format(c Comment) Comment {
c.Text = f.FormatText(c.Text)
return c
}
// FormatText formatting line
func (f *CommentFormater) FormatText(txt string) (res string) {
mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
blackfriday.BackslashLineBreak | blackfriday.Autolink
res = string(blackfriday.Run([]byte(txt), blackfriday.WithExtensions(mdExt)))
for _, conv := range f.converters {
res = conv.Convert(res)
}
res = f.shortenAutoLinks(res, shortURLLen)
return res
}
// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
func (f *CommentFormater) shortenAutoLinks(commentHTML string, max int) (resHTML string) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
if err != nil {
return commentHTML
}
doc.Find("a").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Attr("href"); ok {
if href != s.Text() || len(href) < max+3 || max < 3 {
return
}
url, e := url.Parse(href)
if e != nil {
return
}
url.Path, url.RawQuery, url.Fragment = "", "", ""
host := url.String()
if host == "" {
return
}
short := href[:max-3]
if len(short) < len(host) {
short = host
}
s.SetText(short + "...")
}
})
resHTML, err = doc.Find("body").Html()
if err != nil {
return commentHTML
}
return resHTML
}
backend/app/store/formater_test.go
+98
View File
@@ -0,0 +1,98 @@
package store
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
type mockConvertor struct{}
func (m mockConvertor) Convert(text string) string { return text + "!converted" }
func TestFormater_FormatText(t *testing.T) {
tbl := []struct {
in, out string
}{
{"", "!converted"},
{"12345 abc", "<p>12345 abc</p>\n!converted"},
{"**xyz** _aaa_", "<p><strong>xyz</strong> <em>aaa</em></p>\n!converted"},
{
"http://127.0.0.1/some-long-link/12345/678901234567890", "<p><a href=\"http://127.0.0.1/some-long-link/12345/678901234567890\">http://127.0.0.1/some-long-link/12345/6789012...</a></p>\n!converted",
},
}
f := NewCommentFormater(mockConvertor{})
for n, tt := range tbl {
assert.Equal(t, tt.out, f.FormatText(tt.in), "check #%d", n)
}
}
func TestFormater_FormatTextNoConvertor(t *testing.T) {
f := NewCommentFormater()
assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345"))
}
func TestFormater_FormatComment(t *testing.T) {
comment := Comment{
Text: `blah`,
User: User{ID: "username"},
ParentID: "p123",
ID: "123",
Locator: Locator{SiteID: "site", URL: "url"},
Score: 10,
Pin: true,
Deleted: true,
Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
Votes: map[string]bool{"uu": true},
}
f := NewCommentFormater(mockConvertor{})
exp := comment
exp.Text = "<p>blah</p>\n!converted"
assert.Equal(t, exp, f.Format(comment))
}
func TestFormater_ShortenAutoLinks(t *testing.T) {
f := NewCommentFormater(nil)
tbl := []struct {
max int
in, out string
}{
{32, "", ""},
{32, "text", "text"},
{32, "<p>asd</p>", "<p>asd</p>"},
{5, `<a href="incorrect-url">incorrect-url</a>`, `<a href="incorrect-url">incorrect-url</a>`},
{32, `<a href="https://blah.com">some text, not href</a>`, `<a href="https://blah.com">some text, not href</a>`},
{
32,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
{
31,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=1...</a>`,
},
{
15,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
3,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
-1,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
}
for n, tt := range tbl {
got := f.shortenAutoLinks(tt.in, tt.max)
assert.Equalf(t, tt.out, got, "check #%d", n)
}
}
backend/app/store/service/service.go
-1
View File
@@ -43,7 +43,6 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
if comment.Votes == nil {
comment.Votes = make(map[string]bool)
}
comment.Sanitize() // clear potentially dangerous js from all parts of comment
secret, err := s.KeyStore.Get(comment.Locator.SiteID)