mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

import from commento engine

Browse Source
This commit is contained in:
romanilchyshyn
2021-10-06 13:51:31 -05:00
committed by Umputun
parent 5abeab4008
commit e341227cae
12 changed files with 323 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5
View File
@@ -367,6 +367,11 @@ Remark42 supports importing comments from Disqus, WordPress, or native backup fo
2. Move this file to your Remark42 host within `./var`
3. Run import command - `docker exec -it remark42 import -p wordpress -f /srv/var/{wordpress-export-name}.xml -s {your site ID}`
##### Initial import from Commento
1. Move exported json file to your Remark42 host within `./var`
2. Run import command - `docker exec -it remark42 import -p commento -f /srv/var/{commento-export-name}.json -s {your site ID}`
#### Backup and restore
##### Automatic backups
backend/app/cmd/import.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
// ImportCommand set of flags and command for import
type ImportCommand struct {
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` //nolint
Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" choice:"commento" description:"import format"` //nolint
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
backend/app/cmd/server.go
+1
View File
@@ -463,6 +463,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
NativeImporter: &migrator.Native{DataStore: dataService},
DisqusImporter: &migrator.Disqus{DataStore: dataService},
WordPressImporter: &migrator.WordPress{DataStore: dataService},
CommentoImporter: &migrator.Commento{DataStore: dataService},
NativeExporter: &migrator.Native{DataStore: dataService},
URLMapperMaker: migrator.NewURLMapper,
KeyStore: adminStore,
backend/app/migrator/commento.go
+134
View File
@@ -0,0 +1,134 @@
package migrator
import (
"encoding/json"
"io"
"time"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
log "github.com/go-pkgz/lgr"
)
// Commento implements Importer from commento export json
type Commento struct {
DataStore Store
}
// Credit: https://gitlab.com/commento/commento/-/blob/master/api/domain_import_commento.go#L11-L15
type commentoExport struct {
Version int `json:"version"`
Comments []commentoComment `json:"comments"`
Commenters []commentoCommenter `json:"commenters"`
}
// Credit: https://gitlab.com/commento/commento/-/blob/master/api/comment.go#L7-L20
type commentoComment struct {
CommentHex string `json:"commentHex"`
Domain string `json:"domain,omitempty"`
Path string `json:"url,omitempty"`
CommenterHex string `json:"commenterHex"`
Markdown string `json:"markdown"`
HTML string `json:"html"`
ParentHex string `json:"parentHex"`
Score int `json:"score"`
State string `json:"state,omitempty"`
CreationDate time.Time `json:"creationDate"`
Direction int `json:"direction"`
Deleted bool `json:"deleted"`
}
// Credit: https://gitlab.com/commento/commento/-/blob/master/api/commenter.go#L7-L16
type commentoCommenter struct {
CommenterHex string `json:"commenterHex,omitempty"`
Email string `json:"email,omitempty"`
Name string `json:"name"`
Link string `json:"link"`
Photo string `json:"photo"`
Provider string `json:"provider,omitempty"`
JoinDate time.Time `json:"joinDate,omitempty"`
IsModerator bool `json:"isModerator"`
}
// Import comments from Commento and save to store
func (d *Commento) Import(r io.Reader, siteID string) (size int, err error) {
if e := d.DataStore.DeleteAll(siteID); e != nil {
return 0, e
}
commentsCh := d.convert(r, siteID)
failed, passed := 0, 0
for c := range commentsCh {
if _, err = d.DataStore.Create(c); err != nil {
failed++
continue
}
passed++
}
if failed > 0 {
err = errors.Errorf("failed to save %d comments", failed)
if passed == 0 {
err = errors.New("import failed")
}
}
log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
return passed, err
}
func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
commentsCh := make(chan store.Comment)
decoder := json.NewDecoder(r)
go func() {
var exportedData commentoExport
err := decoder.Decode(&exportedData)
if err != nil {
log.Printf("[WARN] can't decode commento export json, %s", err.Error())
}
usersMap := map[string]store.User{}
for _, commenter := range exportedData.Commenters {
usersMap[commenter.CommenterHex] = store.User{
Name: commenter.Name,
ID: "commento_" + store.EncodeID(commenter.CommenterHex),
Picture: commenter.Photo,
}
}
for _, comment := range exportedData.Comments {
u, ok := usersMap[comment.CommenterHex]
if !ok {
continue
}
if comment.Deleted {
continue
}
c := store.Comment{
ID: comment.CommentHex,
Locator: store.Locator{
URL: comment.Path,
SiteID: siteID,
},
User: u,
Text: comment.Markdown,
Timestamp: comment.CreationDate,
ParentID: comment.ParentHex,
Imported: true,
}
commentsCh <- c
}
close(commentsCh)
}()
return commentsCh
}
backend/app/migrator/commento_test.go
+53
View File
@@ -0,0 +1,53 @@
package migrator
import (
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/service"
bolt "go.etcd.io/bbolt"
)
func TestCommento_Import(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.NoError(t, err, "create store")
dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
defer dataStore.Close()
d := Commento{DataStore: &dataStore}
fh, err := os.Open("testdata/commento.json")
require.NoError(t, err)
size, err := d.Import(fh, "test")
assert.NoError(t, err)
assert.Equal(t, 2, size)
last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
assert.NoError(t, err)
require.Equal(t, 2, len(last), "2 comments imported")
t.Log(last[0])
c := last[0] // last reverses, get first one
assert.Equal(t, "Great reply!", c.Text)
assert.Equal(t, "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9", c.ID)
assert.Equal(t, "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854", c.ParentID)
assert.Equal(t, store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"}, c.Locator)
assert.Equal(t, "Saturnin Uf", c.User.Name)
assert.Equal(t, "commento_35369aeb6ac5255de30410a0f86dc71eb9c6d0ca", c.User.ID)
assert.True(t, c.Imported)
posts, err := dataStore.List("test", 0, 0)
assert.NoError(t, err)
assert.Equal(t, 1, len(posts), "1 post")
count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"})
assert.NoError(t, err)
assert.Equal(t, 2, count)
}
backend/app/migrator/migrator.go
+2
View File
@@ -64,6 +64,8 @@ func ImportComments(p ImportParams) (int, error) {
importer = &Disqus{DataStore: p.DataStore}
case "wordpress":
importer = &WordPress{DataStore: p.DataStore}
case "commento":
importer = &Commento{DataStore: p.DataStore}
case "native":
importer = &Native{DataStore: p.DataStore}
default:
backend/app/migrator/migrator_test.go
+21
View File
@@ -64,6 +64,27 @@ func TestMigrator_ImportWordPress(t *testing.T) {
assert.Equal(t, 3, len(last), "3 comments imported")
}
func TestMigrator_ImportCommento(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.NoError(t, err, "create store")
dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
defer dataStore.Close()
size, err := ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "testdata/commento.json",
SiteID: "test",
Provider: "commento",
})
assert.NoError(t, err)
assert.Equal(t, 2, size)
last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
assert.NoError(t, err)
assert.Equal(t, 2, len(last), "2 comments imported")
}
func TestMigrator_ImportNative(t *testing.T) {
defer func() {
os.Remove("/tmp/remark-test.db")
backend/app/migrator/testdata/commento.json
Vendored
+69
View File
@@ -0,0 +1,69 @@
{
"version": 1,
"comments": [
{
"commentHex": "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945",
"domain": "example.com",
"url": "https://example.com/blog/post/1",
"commenterHex": "anonymous",
"markdown": "Example comment created by user.",
"html": "",
"parentHex": "root",
"score": 1,
"state": "approved",
"creationDate": "2021-03-12T11:21:56Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
"domain": "example.com",
"url": "https://example.com/blog/post/1",
"commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
"markdown": "Example 2 comment created by user.",
"html": "",
"parentHex": "root",
"score": 0,
"state": "approved",
"creationDate": "2021-03-17T12:09:47.722181Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9",
"domain": "example.com",
"url": "https://example.com/blog/post/1",
"commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
"markdown": "Great reply!",
"html": "",
"parentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
"score": 0,
"state": "approved",
"creationDate": "2021-05-11T15:43:01.852651Z",
"direction": 0,
"deleted": false
}
],
"commenters": [
{
"commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
"email": "somegreatmail@gmail.com",
"name": "User5276",
"link": "https://example.com/profile/257",
"photo": "https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
"provider": "sso:example.com",
"joinDate": "2021-03-19T19:27:25.954285Z",
"isModerator": false
},
{
"commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
"email": "moregreatmail@gmail.com",
"name": "Saturnin Uf",
"link": "https://example.com/profile/259",
"photo": "https://secure.gravatar.com/avatar/6481228d190f0286a42bee9041f9b1a1",
"provider": "sso:example.com",
"joinDate": "2021-03-21T12:15:37.536035Z",
"isModerator": false
}
]
}
backend/app/rest/api/migrator.go
+3
View File
@@ -27,6 +27,7 @@ type Migrator struct {
NativeImporter migrator.Importer
DisqusImporter migrator.Importer
WordPressImporter migrator.Importer
CommentoImporter migrator.Importer
NativeExporter migrator.Exporter
URLMapperMaker migrator.MapperMaker
KeyStore KeyStore
@@ -228,6 +229,8 @@ func (m *Migrator) runImport(siteID, provider, tmpfile string) {
importer = m.DisqusImporter
case "wordpress":
importer = m.WordPressImporter
case "commento":
importer = m.CommentoImporter
default:
importer = m.NativeImporter
}
backend/app/rest/api/migrator_test.go
+28
View File
@@ -109,6 +109,34 @@ func TestMigrator_ImportFromWP(t *testing.T) {
waitForMigrationCompletion(t, ts)
}
func TestMigrator_ImportFromCommento(t *testing.T) {
ts, _, teardown := startupT(t)
defer teardown()
r := strings.NewReader(`{"version":1,"comments":[{"commentHex":"7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
"domain":"example.com","url":"https://example.com/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
"markdown":"Example content","html":"","parentHex":"root","score":0,"state":"approved","creationDate":"2021-03-17T12:09:47.722181Z",
"direction":0,"deleted":false}],"commenters":[{"commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
"email":"somegreatmail@gmail.com","name":"User5276","link":"https://example.com/profile/257","photo":"https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
"provider":"sso:example.com","joinDate":"2021-03-19T19:27:25.954285Z","isModerator":false}]}`)
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
assert.NoError(t, err)
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.SetBasicAuth("admin", "password")
resp, err := client.Do(req)
assert.NoError(t, err)
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
assert.NoError(t, resp.Body.Close())
waitForMigrationCompletion(t, ts)
}
func TestMigrator_ImportRejected(t *testing.T) {
ts, _, teardown := startupT(t)
defer teardown()
backend/app/rest/api/rest_test.go
+1
View File
@@ -430,6 +430,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
Migrator: &Migrator{
DisqusImporter: &migrator.Disqus{DataStore: dataStore},
WordPressImporter: &migrator.WordPress{DataStore: dataStore},
CommentoImporter: &migrator.Commento{DataStore: dataStore},
NativeImporter: &migrator.Native{DataStore: dataStore},
NativeExporter: &migrator.Native{DataStore: dataStore},
URLMapperMaker: migrator.NewURLMapper,
site/src/docs/backup/migration/index.md
+5
View File
@@ -15,3 +15,8 @@ Remark42 supports importing comments from Disqus, WordPress, or native backup fo
1. Use [that instruction](https://wordpress.com/support/export/) to export comments to file using standard WordPress functionality
2. Move this file to your Remark42 host within `./var`
3. Run import command - `docker exec -it remark42 import -p wordpress -f /srv/var/{wordpress-export-name}.xml -s {your site ID}`
### Import from Commento
1. Move exported json file to your Remark42 host within `./var`
2. Run import command - `docker exec -it remark42 import -p commento -f /srv/var/{commento-export-name}.json -s {your site ID}`