diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go index e9d657b3..ca6b2c5f 100644 --- a/backend/app/migrator/disqus.go +++ b/backend/app/migrator/disqus.go @@ -133,7 +133,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { Name: comment.AuthorName, IP: comment.IP, }, - Text: cleanText(comment.Message), + Text: d.cleanText(comment.Message), Timestamp: comment.CreatedAt, ParentID: comment.Pid.Val, } @@ -158,7 +158,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { return commentsCh } -func cleanText(text string) string { +func (*Disqus) cleanText(text string) string { text = strings.Replace(text, "\n", "", -1) text = strings.Replace(text, "\t", "", -1) return text diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go index 362ef881..5b2913b8 100644 --- a/backend/app/migrator/wordpress.go +++ b/backend/app/migrator/wordpress.go @@ -52,6 +52,13 @@ func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { return err } +// wpCommentConverter implements store.CommentConverter +type wpCommentConverter struct{} + +func (wpCommentConverter) Convert(text string) string { + return html.UnescapeString(text) // sanitize remains on comment create +} + // Import comments from WP and save to store func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) { @@ -92,6 +99,9 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { rejectedComments int // not approved }{} + commentConverter := new(wpCommentConverter) + commentFormatter := store.NewCommentFormatter(commentConverter) + go func() { for { t, err := decoder.Token() @@ -119,6 +129,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { if comment.PID == "0" { comment.PID = "" } + c := store.Comment{ ID: comment.ID, Locator: store.Locator{URL: item.Link, SiteID: siteID}, @@ -127,11 +138,11 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { Name: comment.Author, IP: comment.AuthorIP, }, - Text: w.cleanUnescapeText(comment.Content), // sanitize remains on comment create + Text: comment.Content, Timestamp: comment.Date.time, ParentID: comment.PID, } - commentsCh <- c + commentsCh <- commentFormatter.Format(c) stats.inpComments++ if stats.inpComments%1000 == 0 { log.Printf("[DEBUG] proccessed %d comments", stats.inpComments) @@ -146,9 +157,3 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { }() return commentsCh } - -func (w *WordPress) cleanUnescapeText(text string) string { - text = cleanText(text) - text = html.UnescapeString(text) - return text -} diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go index d1cef135..ae71e4d3 100644 --- a/backend/app/migrator/wordpress_test.go +++ b/backend/app/migrator/wordpress_test.go @@ -39,7 +39,7 @@ func TestWordPress_Import(t *testing.T) { assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP) ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14") assert.Equal(t, ts, c.Timestamp) - assert.Equal(t, c.Text, "Mekkatorque was over in that tent up to the right") + assert.Equal(t, c.Text, "

Mekkatorque was over in that tent up to the right

\n") posts, err := dataStore.List(siteID, 0, 0) assert.Nil(t, err) @@ -69,7 +69,7 @@ func TestWordPress_Convert(t *testing.T) { SiteID: "testWP", URL: "https://realmenweardress.es/2010/07/do-you-rp/", }, - Text: `[...] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. [...]`, + Text: `

[…] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. […]

` + "\n", User: store.User{ Name: "Wednesday Reading « Cynwise's Battlefield Manual", ID: "wordpress_" + store.EncodeID("Wednesday Reading « Cynwise's Battlefield Manual"), @@ -80,6 +80,25 @@ func TestWordPress_Convert(t *testing.T) { assert.Equal(t, exp1, comments[1]) } +func TestWP_Convert_MD(t *testing.T) { + wp := WordPress{} + ch := wp.convert(strings.NewReader(xmlTestWPmd), "siteID") + + comments := []store.Comment{} + for c := range ch { + comments = append(comments, c) + } + assert.Equal(t, 3, len(comments), "3 comments exported") + + assert.Equal(t, "

Row1
\nRow2

\n\n

Row4

\n", comments[0].Text) + + assert.Equal(t, "

markdown text

\n", comments[1].Text) + + expText := `

Row1 Link http://releases.rancher.com/os/latest markdown text blah

` + expText += "\n\n

Row3 markdownmd block

\n" + assert.Equal(t, expText, comments[2].Text) +} + var xmlTestWP = ` ` + +// parts of unused xml tags are omitted +var xmlTestWPmd = ` + + + + Deploying RancherOS on Vultr instances + https://realmenweardress.es/2016/07/deploying-rancheros-on-vultr-instances/ + + + 1 + + + https://eric.com + + + + + + + 0 + 0 + + + + 2 + + + https://eric.com + + + + + + + 0 + 0 + + + + 2 + + + https://eric.com + + + + + + + 0 + 0 + + + + + +`