diff --git a/README.md b/README.md index d49d7053..0558e2aa 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * Social login via Google, Facebook, Github and Yandex * Multi-level nested comments with both tree and plain presentations -* Import from disqus +* Import from disqus, wordpress * Markdown support * Moderator can remove comments and block users * Voting, pinning and verification system @@ -162,6 +162,12 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co 2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip .xml.gz`. 3. Run import command - `docker-compose exec remark42 /srv/import-disqus.sh .xml ` +#### Initial import from WordPress + +1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. +2. Move this file to your remark42 host within `./var` +3. Run import command - `docker-compose exec remark42 /srv/import-wordpress.sh .xml ` + #### Backup and restore ##### Automatic backups diff --git a/backend/app/main.go b/backend/app/main.go index 0e1d1aa3..a88b5759 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -196,12 +196,13 @@ func New(opts Opts) (*Application, error) { exporter := &migrator.Remark{DataStore: dataService} migr := &api.Migrator{ - Version: revision, - Cache: loadingCache, - NativeImporter: &migrator.Remark{DataStore: dataService}, - DisqusImporter: &migrator.Disqus{DataStore: dataService}, - NativeExported: &migrator.Remark{DataStore: dataService}, - SecretKey: opts.SecretKey, + Version: revision, + Cache: loadingCache, + NativeImporter: &migrator.Remark{DataStore: dataService}, + DisqusImporter: &migrator.Disqus{DataStore: dataService}, + WordPressImporter: &migrator.WordPress{DataStore: dataService}, + NativeExported: &migrator.Remark{DataStore: dataService}, + SecretKey: opts.SecretKey, } authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go index 547fccda..e9d657b3 100644 --- a/backend/app/migrator/disqus.go +++ b/backend/app/migrator/disqus.go @@ -68,14 +68,14 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) { } if failed > 0 { - return passed, errors.Errorf("failed to save %d comments", failed) + err = errors.Errorf("failed to save %d comments", failed) + if passed == 0 { + err = errors.New("import failed") + } } log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID) - if failed > 0 && passed == 0 { - err = errors.New("import failed") - } return passed, err } @@ -133,7 +133,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { Name: comment.AuthorName, IP: comment.IP, }, - Text: d.cleanText(comment.Message), + Text: cleanText(comment.Message), Timestamp: comment.CreatedAt, ParentID: comment.Pid.Val, } @@ -158,7 +158,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { return commentsCh } -func (d *Disqus) cleanText(text string) string { +func cleanText(text string) string { text = strings.Replace(text, "\n", "", -1) text = strings.Replace(text, "\t", "", -1) return text diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index 96142a5d..af7a55ba 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -21,7 +21,7 @@ func TestDisqus_Import(t *testing.T) { require.Nil(t, err, "create store") dataStore := service.DataStore{Interface: b} d := Disqus{DataStore: &dataStore} - size, err := d.Import(strings.NewReader(xmlTest), "test") + size, err := d.Import(strings.NewReader(xmlTestDisqus), "test") assert.Nil(t, err) assert.Equal(t, 3, size) @@ -49,7 +49,7 @@ func TestDisqus_Import(t *testing.T) { func TestDisqus_Convert(t *testing.T) { d := Disqus{} - ch := d.convert(strings.NewReader(xmlTest), "test") + ch := d.convert(strings.NewReader(xmlTestDisqus), "test") res := []store.Comment{} for comment := range ch { @@ -74,7 +74,7 @@ func TestDisqus_Convert(t *testing.T) { assert.Equal(t, exp0, res[0]) } -var xmlTest = ` +var xmlTestDisqus = ` diff --git a/backend/app/migrator/migrator.go b/backend/app/migrator/migrator.go index 40c156ec..422a25ab 100644 --- a/backend/app/migrator/migrator.go +++ b/backend/app/migrator/migrator.go @@ -46,6 +46,8 @@ func ImportComments(p ImportParams) (int, error) { switch p.Provider { case "disqus": importer = &Disqus{DataStore: p.DataStore} + case "wordpress": + importer = &WordPress{DataStore: p.DataStore} case "native": importer = &Remark{DataStore: p.DataStore} default: diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 8825c53e..120b1185 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -19,7 +19,7 @@ func TestMigrator_ImportDisqus(t *testing.T) { os.Remove("/tmp/disqus-test.xml") }() - err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTest), 0600) + err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600) require.Nil(t, err) b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) @@ -39,6 +39,32 @@ func TestMigrator_ImportDisqus(t *testing.T) { assert.Equal(t, 3, len(last), "3 comments imported") } +func TestMigrator_ImportWordPress(t *testing.T) { + defer func() { + os.Remove("/tmp/remark-test.db") + os.Remove("/tmp/wordpress-test.xml") + }() + + err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600) + require.Nil(t, err) + + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + require.Nil(t, err, "create store") + dataStore := &service.DataStore{Interface: b} + size, err := ImportComments(ImportParams{ + DataStore: dataStore, + InputFile: "/tmp/wordpress-test.xml", + SiteID: "test", + Provider: "wordpress", + }) + assert.Nil(t, err) + assert.Equal(t, 3, size) + + last, err := dataStore.Last("test", 10) + assert.Nil(t, err) + assert.Equal(t, 3, len(last), "3 comments imported") +} + func TestMigrator_ImportRemark(t *testing.T) { defer func() { os.Remove("/tmp/remark-test.db") @@ -70,7 +96,7 @@ func TestMigrator_ImportRemark(t *testing.T) { } func TestMigrator_ImportFailed(t *testing.T) { - + defer os.Remove("/tmp/remark-test.db") b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") dataStore := &service.DataStore{Interface: b} diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go new file mode 100644 index 00000000..362ef881 --- /dev/null +++ b/backend/app/migrator/wordpress.go @@ -0,0 +1,154 @@ +package migrator + +import ( + "encoding/xml" + "html" + "io" + "log" + "time" + + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store" +) + +const wpTimeLayout = "2006-01-02 15:04:05" + +// WordPress implements Importer from WP xml +type WordPress struct { + DataStore Store +} + +type wpItem struct { + Link string `xml:"link"` + Comments []wpComment `xml:"comment"` +} + +type wpComment struct { + ID string `xml:"comment_id"` + Author string `xml:"comment_author"` + AuthorEmail string `xml:"comment_author_email"` + AuthorIP string `xml:"comment_author_IP"` + Date wpTime `xml:"comment_date_gmt"` + Content string `xml:"comment_content"` + Approved string `xml:"comment_approved"` + PID string `xml:"comment_parent"` +} + +type wpTime struct { + time time.Time +} + +func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var v string + if err := d.DecodeElement(&v, &start); err != nil { + return err + } + t, err := time.Parse(wpTimeLayout, v) + if err != nil { + return err + } + w.time = t + return err +} + +// Import comments from WP and save to store +func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) { + + if err = w.DataStore.DeleteAll(siteID); err != nil { + return 0, err + } + + commentsCh := w.convert(r, siteID) + failed, passed := 0, 0 + for c := range commentsCh { + if _, err = w.DataStore.Create(c); err != nil { + failed++ + continue + } + passed++ + } + + if failed > 0 { + err = errors.Errorf("failed to save %d comments", failed) + if passed == 0 { + err = errors.New("import failed") + } + } + + log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID) + + return passed, err +} + +func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { + + decoder := xml.NewDecoder(r) + commentsCh := make(chan store.Comment) + + stats := struct { + inpItems, failedItems int + inpComments, failedComments int + rejectedComments int // not approved + }{} + + go func() { + for { + t, err := decoder.Token() + if t == nil || err != nil { + break + } + + switch el := t.(type) { + case xml.StartElement: + if el.Name.Local == "item" { + stats.inpItems++ + item := wpItem{} + if err := decoder.DecodeElement(&item, &el); err != nil { + log.Printf("[WARN] Can't decode item, %s", err) + stats.failedItems++ + continue + } + if item.Comments != nil { + for _, comment := range item.Comments { + if comment.Approved != "1" { + stats.rejectedComments++ + continue + } + + if comment.PID == "0" { + comment.PID = "" + } + c := store.Comment{ + ID: comment.ID, + Locator: store.Locator{URL: item.Link, SiteID: siteID}, + User: store.User{ + ID: "wordpress_" + store.EncodeID(comment.Author), + Name: comment.Author, + IP: comment.AuthorIP, + }, + Text: w.cleanUnescapeText(comment.Content), // sanitize remains on comment create + Timestamp: comment.Date.time, + ParentID: comment.PID, + } + commentsCh <- c + stats.inpComments++ + if stats.inpComments%1000 == 0 { + log.Printf("[DEBUG] proccessed %d comments", stats.inpComments) + } + } + } + } + } + } + close(commentsCh) + log.Printf("[INFO] converted %d comments, %+v", stats.inpComments-stats.failedComments, stats) + }() + return commentsCh +} + +func (w *WordPress) cleanUnescapeText(text string) string { + text = cleanText(text) + text = html.UnescapeString(text) + return text +} diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go new file mode 100644 index 00000000..c915956a --- /dev/null +++ b/backend/app/migrator/wordpress_test.go @@ -0,0 +1,235 @@ +package migrator + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/coreos/bbolt" + "github.com/stretchr/testify/assert" + + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/service" +) + +func TestWordPress_Import(t *testing.T) { + siteID := "testWP" + defer os.Remove("/tmp/remark-test.db") + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) + assert.Nil(t, err, "create store") + + dataStore := service.DataStore{Interface: b} + wp := WordPress{DataStore: &dataStore} + size, err := wp.Import(strings.NewReader(xmlTestWP), siteID) + assert.Nil(t, err) + assert.Equal(t, 3, size) + + last, err := dataStore.Last(siteID, 10) + assert.Nil(t, err) + assert.Equal(t, 3, len(last), "3 comments imported") + + c := last[0] + assert.Equal(t, "14", c.ID) + assert.Equal(t, store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID}, c.Locator) + assert.Equal(t, "wordpress_75b2b81081f82495d7af26759e67af6554ffda4a", c.User.ID) + assert.Equal(t, "SuperUser3", c.User.Name) + assert.Equal(t, "b646e160768fbc1414d3b2c8f88a767bfbb00871", c.User.IP) + ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14") + assert.Equal(t, ts, c.Timestamp) + assert.Equal(t, c.Text, "Mekkatorque was over in that tent up to the right") + + posts, err := dataStore.List(siteID, 0, 0) + assert.Nil(t, err) + assert.Equal(t, 1, len(posts)) + + p := posts[0] + assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL) + + count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID}) + assert.Nil(t, err) + assert.Equal(t, 3, count) +} + +func TestWordPress_Convert(t *testing.T) { + wp := WordPress{} + ch := wp.convert(strings.NewReader(xmlTestWP), "testWP") + + comments := []store.Comment{} + for c := range ch { + comments = append(comments, c) + } + assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded") + + exp1 := store.Comment{ + ID: "13", + Locator: store.Locator{ + SiteID: "testWP", + URL: "https://realmenweardress.es/2010/07/do-you-rp/", + }, + Text: `[...] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. [...]`, + User: store.User{ + Name: "Wednesday Reading « Cynwise's Battlefield Manual", + ID: "wordpress_" + store.EncodeID("Wednesday Reading « Cynwise's Battlefield Manual"), + IP: "74.200.244.101", + }, + } + exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08") + assert.Equal(t, exp1, comments[1]) +} + +var xmlTestWP = ` + + + + + Real Men Wear Dress.es + https://realmenweardress.es + SuperAdmin's gaming and technological musings + Mon, 23 Jul 2018 10:21:47 +0000 + en-US + 1.2 + https://realmenweardress.es + https://realmenweardress.es + + 2 + 1 + + + 25 + + + + + + + 39 + + + + + https://wordpress.org/?v=4.8.1 + + + Post without comments + https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/ + Sat, 19 Jun 2010 08:34:13 +0000 + + http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg + + + + 6 + + + + + + + 1 + 0 + + + 0 + + + + + + + + Post with comments. One is not approved + https://realmenweardress.es/2010/07/do-you-rp/ + Mon, 19 Jul 2010 14:24:22 +0000 + + http://realmenweardress.es/?p=100 + + I need to stand on things else I can't reachMeet Grokknomel?]]> + + 100 + + + + + + + 0 + 0 + + + 0 + + + + + + + + + + 8 + + + http://superuser1.blogspot.com + + + + + + + 0 + 0 + + + 9 + + + http://thewowstorm.wordpress.com + + + + + + + 0 + 0 + + + 13 + + + http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/ + + + + + + + 0 + 0 + + + 14 + + + http://realmenweardress.es + + + + + + + 13 + 2 + + + + +` diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index c8bba0da..bb891bd2 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -23,12 +23,13 @@ import ( // Migrator rest runs on unexposed port and available for local requests only type Migrator struct { - Version string - Cache cache.LoadingCache - NativeImporter migrator.Importer - DisqusImporter migrator.Importer - NativeExported migrator.Exporter - SecretKey string + Version string + Cache cache.LoadingCache + NativeImporter migrator.Importer + DisqusImporter migrator.Importer + WordPressImporter migrator.Importer + NativeExported migrator.Exporter + SecretKey string httpServer *http.Server lock sync.Mutex @@ -76,7 +77,7 @@ func (m *Migrator) routes() chi.Router { return router } -// POST /import?secret=key&site=site-id&provider=disqus|remark +// POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress // imports comments from post body. func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { @@ -88,10 +89,17 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { } siteID := r.URL.Query().Get("site") - importer := m.NativeImporter - if r.URL.Query().Get("provider") == "disqus" { + + var importer migrator.Importer + switch r.URL.Query().Get("provider") { + case "disqus": importer = m.DisqusImporter + case "wordpress": + importer = m.WordPressImporter + default: + importer = m.NativeImporter } + log.Printf("[DEBUG] import request for site=%s", siteID) size, err := importer.Import(r.Body, siteID) if err != nil { diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 98a34908..43f6c0e4 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -40,6 +40,26 @@ func TestMigrator_Import(t *testing.T) { assert.Equal(t, `{"size":2,"status":"ok"}`+"\n", string(b)) } +func TestMigrator_ImportFromWP(t *testing.T) { + srv, ts := prepImportSrv(t) + assert.NotNil(t, srv) + defer cleanupImportSrv(srv, ts) + + r := strings.NewReader(xmlTestWP) + + client := &http.Client{Timeout: 1 * time.Second} + req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress&secret=123456", r) + assert.Nil(t, err) + req.Header.Add("Content-Type", "application/xml; charset=utf-8") + resp, err := client.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + b, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, `{"size":3,"status":"ok"}`+"\n", string(b)) +} + func TestMigrator_ImportRejected(t *testing.T) { srv, ts := prepImportSrv(t) assert.NotNil(t, srv) @@ -109,11 +129,12 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) { require.Nil(t, err) dataStore := &service.DataStore{Interface: b} svc = &Migrator{ - DisqusImporter: &migrator.Disqus{DataStore: dataStore}, - NativeImporter: &migrator.Remark{DataStore: dataStore}, - NativeExported: &migrator.Remark{DataStore: dataStore}, - Cache: &cache.Nop{}, - SecretKey: "123456", + DisqusImporter: &migrator.Disqus{DataStore: dataStore}, + WordPressImporter: &migrator.WordPress{DataStore: dataStore}, + NativeImporter: &migrator.Remark{DataStore: dataStore}, + NativeExported: &migrator.Remark{DataStore: dataStore}, + Cache: &cache.Nop{}, + SecretKey: "123456", } routes := svc.routes() @@ -125,3 +146,158 @@ func cleanupImportSrv(srv *Migrator, ts *httptest.Server) { ts.Close() os.Remove(testDb) } + +var xmlTestWP = ` + + + + + Real Men Wear Dress.es + https://realmenweardress.es + SuperAdmin's gaming and technological musings + Mon, 23 Jul 2018 10:21:47 +0000 + en-US + 1.2 + https://realmenweardress.es + https://realmenweardress.es + + 2 + 1 + + + 25 + + + + + + + 39 + + + + + https://wordpress.org/?v=4.8.1 + + + Post without comments + https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/ + Sat, 19 Jun 2010 08:34:13 +0000 + + http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg + + + + 6 + + + + + + + 1 + 0 + + + 0 + + + + + + + + Post with comments. One is not approved + https://realmenweardress.es/2010/07/do-you-rp/ + Mon, 19 Jul 2010 14:24:22 +0000 + + http://realmenweardress.es/?p=100 + + I need to stand on things else I can't reachMeet Grokknomel?]]> + + 100 + + + + + + + 0 + 0 + + + 0 + + + + + + + + + + 8 + + + http://superuser1.blogspot.com + + + + + + + 0 + 0 + + + 9 + + + http://thewowstorm.wordpress.com + + + + + + + 0 + 0 + + + 13 + + + http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/ + + + + + + + 0 + 0 + + + 14 + + + http://realmenweardress.es + + + + + + + 13 + 2 + + + + +` diff --git a/backend/scripts/import-wordpress.sh b/backend/scripts/import-wordpress.sh new file mode 100755 index 00000000..1c224379 --- /dev/null +++ b/backend/scripts/import-wordpress.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e +echo "import wordpress file $1 to site $2" +curl -X POST -H "Content-Type: application/xml" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=wordpress&secret=${SECRET}" +echo "import completed" \ No newline at end of file