diff --git a/README.md b/README.md index b9827f29..c0b80ef1 100644 --- a/README.md +++ b/README.md @@ -367,6 +367,11 @@ Remark42 supports importing comments from Disqus, WordPress, or native backup fo 2. Move this file to your Remark42 host within `./var` 3. Run import command - `docker exec -it remark42 import -p wordpress -f /srv/var/{wordpress-export-name}.xml -s {your site ID}` +##### Initial import from Commento + +1. Move exported json file to your Remark42 host within `./var` +2. Run import command - `docker exec -it remark42 import -p commento -f /srv/var/{commento-export-name}.json -s {your site ID}` + #### Backup and restore ##### Automatic backups diff --git a/backend/app/cmd/import.go b/backend/app/cmd/import.go index 05e5c76f..ccc9b5d5 100644 --- a/backend/app/cmd/import.go +++ b/backend/app/cmd/import.go @@ -18,7 +18,7 @@ import ( // ImportCommand set of flags and command for import type ImportCommand struct { InputFile string `short:"f" long:"file" description:"input file name" required:"true"` - Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` //nolint + Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" choice:"commento" description:"import format"` //nolint Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"` diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 723459ed..525e0ae7 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -463,6 +463,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) { NativeImporter: &migrator.Native{DataStore: dataService}, DisqusImporter: &migrator.Disqus{DataStore: dataService}, WordPressImporter: &migrator.WordPress{DataStore: dataService}, + CommentoImporter: &migrator.Commento{DataStore: dataService}, NativeExporter: &migrator.Native{DataStore: dataService}, URLMapperMaker: migrator.NewURLMapper, KeyStore: adminStore, diff --git a/backend/app/migrator/commento.go b/backend/app/migrator/commento.go new file mode 100644 index 00000000..d4a46a15 --- /dev/null +++ b/backend/app/migrator/commento.go @@ -0,0 +1,134 @@ +package migrator + +import ( + "encoding/json" + "io" + "time" + + "github.com/pkg/errors" + "github.com/umputun/remark42/backend/app/store" + + log "github.com/go-pkgz/lgr" +) + +// Commento implements Importer from commento export json +type Commento struct { + DataStore Store +} + +// Credit: https://gitlab.com/commento/commento/-/blob/master/api/domain_import_commento.go#L11-L15 +type commentoExport struct { + Version int `json:"version"` + Comments []commentoComment `json:"comments"` + Commenters []commentoCommenter `json:"commenters"` +} + +// Credit: https://gitlab.com/commento/commento/-/blob/master/api/comment.go#L7-L20 +type commentoComment struct { + CommentHex string `json:"commentHex"` + Domain string `json:"domain,omitempty"` + Path string `json:"url,omitempty"` + CommenterHex string `json:"commenterHex"` + Markdown string `json:"markdown"` + HTML string `json:"html"` + ParentHex string `json:"parentHex"` + Score int `json:"score"` + State string `json:"state,omitempty"` + CreationDate time.Time `json:"creationDate"` + Direction int `json:"direction"` + Deleted bool `json:"deleted"` +} + +// Credit: https://gitlab.com/commento/commento/-/blob/master/api/commenter.go#L7-L16 +type commentoCommenter struct { + CommenterHex string `json:"commenterHex,omitempty"` + Email string `json:"email,omitempty"` + Name string `json:"name"` + Link string `json:"link"` + Photo string `json:"photo"` + Provider string `json:"provider,omitempty"` + JoinDate time.Time `json:"joinDate,omitempty"` + IsModerator bool `json:"isModerator"` +} + +// Import comments from Commento and save to store +func (d *Commento) Import(r io.Reader, siteID string) (size int, err error) { + if e := d.DataStore.DeleteAll(siteID); e != nil { + return 0, e + } + + commentsCh := d.convert(r, siteID) + failed, passed := 0, 0 + for c := range commentsCh { + if _, err = d.DataStore.Create(c); err != nil { + failed++ + continue + } + passed++ + } + + if failed > 0 { + err = errors.Errorf("failed to save %d comments", failed) + if passed == 0 { + err = errors.New("import failed") + } + } + + log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID) + + return passed, err +} + +func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) { + commentsCh := make(chan store.Comment) + + decoder := json.NewDecoder(r) + + go func() { + + var exportedData commentoExport + err := decoder.Decode(&exportedData) + if err != nil { + log.Printf("[WARN] can't decode commento export json, %s", err.Error()) + } + + usersMap := map[string]store.User{} + for _, commenter := range exportedData.Commenters { + usersMap[commenter.CommenterHex] = store.User{ + Name: commenter.Name, + ID: "commento_" + store.EncodeID(commenter.CommenterHex), + Picture: commenter.Photo, + } + } + + for _, comment := range exportedData.Comments { + u, ok := usersMap[comment.CommenterHex] + if !ok { + continue + } + + if comment.Deleted { + continue + } + + c := store.Comment{ + ID: comment.CommentHex, + Locator: store.Locator{ + URL: comment.Path, + SiteID: siteID, + }, + User: u, + Text: comment.Markdown, + Timestamp: comment.CreationDate, + ParentID: comment.ParentHex, + Imported: true, + } + + commentsCh <- c + } + + close(commentsCh) + }() + + return commentsCh +} diff --git a/backend/app/migrator/commento_test.go b/backend/app/migrator/commento_test.go new file mode 100644 index 00000000..fe0212e3 --- /dev/null +++ b/backend/app/migrator/commento_test.go @@ -0,0 +1,53 @@ +package migrator + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/umputun/remark42/backend/app/store" + "github.com/umputun/remark42/backend/app/store/admin" + "github.com/umputun/remark42/backend/app/store/engine" + "github.com/umputun/remark42/backend/app/store/service" + bolt "go.etcd.io/bbolt" +) + +func TestCommento_Import(t *testing.T) { + defer os.Remove("/tmp/remark-test.db") + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + require.NoError(t, err, "create store") + dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")} + defer dataStore.Close() + + d := Commento{DataStore: &dataStore} + fh, err := os.Open("testdata/commento.json") + require.NoError(t, err) + size, err := d.Import(fh, "test") + assert.NoError(t, err) + assert.Equal(t, 2, size) + + last, err := dataStore.Last("test", 10, time.Time{}, adminUser) + assert.NoError(t, err) + require.Equal(t, 2, len(last), "2 comments imported") + + t.Log(last[0]) + + c := last[0] // last reverses, get first one + assert.Equal(t, "Great reply!", c.Text) + assert.Equal(t, "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9", c.ID) + assert.Equal(t, "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854", c.ParentID) + assert.Equal(t, store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"}, c.Locator) + assert.Equal(t, "Saturnin Uf", c.User.Name) + assert.Equal(t, "commento_35369aeb6ac5255de30410a0f86dc71eb9c6d0ca", c.User.ID) + assert.True(t, c.Imported) + + posts, err := dataStore.List("test", 0, 0) + assert.NoError(t, err) + assert.Equal(t, 1, len(posts), "1 post") + + count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"}) + assert.NoError(t, err) + assert.Equal(t, 2, count) +} diff --git a/backend/app/migrator/migrator.go b/backend/app/migrator/migrator.go index fa3c6262..624aeb26 100644 --- a/backend/app/migrator/migrator.go +++ b/backend/app/migrator/migrator.go @@ -64,6 +64,8 @@ func ImportComments(p ImportParams) (int, error) { importer = &Disqus{DataStore: p.DataStore} case "wordpress": importer = &WordPress{DataStore: p.DataStore} + case "commento": + importer = &Commento{DataStore: p.DataStore} case "native": importer = &Native{DataStore: p.DataStore} default: diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index cd63c830..08fc163d 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -64,6 +64,27 @@ func TestMigrator_ImportWordPress(t *testing.T) { assert.Equal(t, 3, len(last), "3 comments imported") } +func TestMigrator_ImportCommento(t *testing.T) { + defer os.Remove("/tmp/remark-test.db") + + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + require.NoError(t, err, "create store") + dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")} + defer dataStore.Close() + size, err := ImportComments(ImportParams{ + DataStore: dataStore, + InputFile: "testdata/commento.json", + SiteID: "test", + Provider: "commento", + }) + assert.NoError(t, err) + assert.Equal(t, 2, size) + + last, err := dataStore.Last("test", 10, time.Time{}, store.User{}) + assert.NoError(t, err) + assert.Equal(t, 2, len(last), "2 comments imported") +} + func TestMigrator_ImportNative(t *testing.T) { defer func() { os.Remove("/tmp/remark-test.db") diff --git a/backend/app/migrator/testdata/commento.json b/backend/app/migrator/testdata/commento.json new file mode 100644 index 00000000..540a64b7 --- /dev/null +++ b/backend/app/migrator/testdata/commento.json @@ -0,0 +1,69 @@ +{ + "version": 1, + "comments": [ + { + "commentHex": "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945", + "domain": "example.com", + "url": "https://example.com/blog/post/1", + "commenterHex": "anonymous", + "markdown": "Example comment created by user.", + "html": "", + "parentHex": "root", + "score": 1, + "state": "approved", + "creationDate": "2021-03-12T11:21:56Z", + "direction": 0, + "deleted": false + }, + { + "commentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854", + "domain": "example.com", + "url": "https://example.com/blog/post/1", + "commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10", + "markdown": "Example 2 comment created by user.", + "html": "", + "parentHex": "root", + "score": 0, + "state": "approved", + "creationDate": "2021-03-17T12:09:47.722181Z", + "direction": 0, + "deleted": false + }, + { + "commentHex": "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9", + "domain": "example.com", + "url": "https://example.com/blog/post/1", + "commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061", + "markdown": "Great reply!", + "html": "", + "parentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854", + "score": 0, + "state": "approved", + "creationDate": "2021-05-11T15:43:01.852651Z", + "direction": 0, + "deleted": false + } + ], + "commenters": [ + { + "commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10", + "email": "somegreatmail@gmail.com", + "name": "User5276", + "link": "https://example.com/profile/257", + "photo": "https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7", + "provider": "sso:example.com", + "joinDate": "2021-03-19T19:27:25.954285Z", + "isModerator": false + }, + { + "commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061", + "email": "moregreatmail@gmail.com", + "name": "Saturnin Uf", + "link": "https://example.com/profile/259", + "photo": "https://secure.gravatar.com/avatar/6481228d190f0286a42bee9041f9b1a1", + "provider": "sso:example.com", + "joinDate": "2021-03-21T12:15:37.536035Z", + "isModerator": false + } + ] +} diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 4f97812c..f61fb038 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -27,6 +27,7 @@ type Migrator struct { NativeImporter migrator.Importer DisqusImporter migrator.Importer WordPressImporter migrator.Importer + CommentoImporter migrator.Importer NativeExporter migrator.Exporter URLMapperMaker migrator.MapperMaker KeyStore KeyStore @@ -228,6 +229,8 @@ func (m *Migrator) runImport(siteID, provider, tmpfile string) { importer = m.DisqusImporter case "wordpress": importer = m.WordPressImporter + case "commento": + importer = m.CommentoImporter default: importer = m.NativeImporter } diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index c5e2c4a8..1a1ab690 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -109,6 +109,34 @@ func TestMigrator_ImportFromWP(t *testing.T) { waitForMigrationCompletion(t, ts) } +func TestMigrator_ImportFromCommento(t *testing.T) { + ts, _, teardown := startupT(t) + defer teardown() + + r := strings.NewReader(`{"version":1,"comments":[{"commentHex":"7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854", +"domain":"example.com","url":"https://example.com/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10", +"markdown":"Example content","html":"","parentHex":"root","score":0,"state":"approved","creationDate":"2021-03-17T12:09:47.722181Z", +"direction":0,"deleted":false}],"commenters":[{"commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10", +"email":"somegreatmail@gmail.com","name":"User5276","link":"https://example.com/profile/257","photo":"https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7", +"provider":"sso:example.com","joinDate":"2021-03-19T19:27:25.954285Z","isModerator":false}]}`) + + client := &http.Client{Timeout: 1 * time.Second} + req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r) + assert.NoError(t, err) + req.Header.Add("Content-Type", "application/json; charset=utf-8") + req.SetBasicAuth("admin", "password") + resp, err := client.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusAccepted, resp.StatusCode) + + b, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) + assert.NoError(t, resp.Body.Close()) + + waitForMigrationCompletion(t, ts) +} + func TestMigrator_ImportRejected(t *testing.T) { ts, _, teardown := startupT(t) defer teardown() diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 6effe33d..c071aceb 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -430,6 +430,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { Migrator: &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, + CommentoImporter: &migrator.Commento{DataStore: dataStore}, NativeImporter: &migrator.Native{DataStore: dataStore}, NativeExporter: &migrator.Native{DataStore: dataStore}, URLMapperMaker: migrator.NewURLMapper, diff --git a/site/src/docs/backup/migration/index.md b/site/src/docs/backup/migration/index.md index b7e9b084..c8426b37 100644 --- a/site/src/docs/backup/migration/index.md +++ b/site/src/docs/backup/migration/index.md @@ -15,3 +15,8 @@ Remark42 supports importing comments from Disqus, WordPress, or native backup fo 1. Use [that instruction](https://wordpress.com/support/export/) to export comments to file using standard WordPress functionality 2. Move this file to your Remark42 host within `./var` 3. Run import command - `docker exec -it remark42 import -p wordpress -f /srv/var/{wordpress-export-name}.xml -s {your site ID}` + +### Import from Commento + +1. Move exported json file to your Remark42 host within `./var` +2. Run import command - `docker exec -it remark42 import -p commento -f /srv/var/{commento-export-name}.json -s {your site ID}`