diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 0ebcd09e..75bc40b5 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -66,13 +66,13 @@ func TestMigrator_ImportWordPress(t *testing.T) { assert.Equal(t, 3, len(last), "3 comments imported") } -func TestMigrator_ImportRemark(t *testing.T) { +func TestMigrator_ImportNative(t *testing.T) { defer func() { os.Remove("/tmp/remark-test.db") os.Remove("/tmp/disqus-test.r42") }() - data := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" + + data := `{"version":1} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" + `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n" err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600) diff --git a/backend/app/migrator/native.go b/backend/app/migrator/native.go index 05e73ce5..abb06eb2 100644 --- a/backend/app/migrator/native.go +++ b/backend/app/migrator/native.go @@ -12,6 +12,8 @@ import ( "github.com/umputun/remark/backend/app/store/service" ) +const natvieVersion = 1 + // Native implements exporter and importer for internal store format // {"version": 1, comments:[{...}\n,{}], meta: {meta}} // each comments starts from the new line @@ -68,7 +70,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) { // exportMeta appends user and post metas to exported stream func (n *Native) exportMeta(siteID string, w io.Writer) (err error) { - m := meta{Version: 1} + m := meta{Version: natvieVersion} m.Users, m.Posts, err = n.DataStore.Metas(siteID) if err != nil { return errors.Wrap(err, "can't get meta") @@ -89,7 +91,11 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID) } - if err := n.DataStore.DeleteAll(siteID); err != nil { + if m.Version != natvieVersion { + return 0, errors.Errorf("unexpected import file version %d", m.Version) + } + + if err = n.DataStore.DeleteAll(siteID); err != nil { return 0, err } @@ -110,7 +116,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { continue } - if _, err := n.DataStore.Create(comment); err != nil { + if _, err = n.DataStore.Create(comment); err != nil { failed++ log.Printf("[WARN] can't write %+v to store, %s", comment, err) continue diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go index 8cadfdfa..46ebcc36 100644 --- a/backend/app/migrator/native_test.go +++ b/backend/app/migrator/native_test.go @@ -100,6 +100,18 @@ func TestNative_Import(t *testing.T) { assert.Equal(t, false, b.IsVerified("radio-t", "user2")) } +func TestNative_ImportWrongVersion(t *testing.T) { + inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]} + {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"} + {"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + + b := prep(t) // write some recs + r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} + size, err := r.Import(strings.NewReader(inp), "radio-t") + assert.EqualError(t, err, "unexpected import file version 2") + assert.Equal(t, 0, size) + +} func TestNative_ImportManyWithError(t *testing.T) { defer os.Remove(testDb) diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index 24fed171..8258cadc 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -455,7 +455,7 @@ func TestAdmin_ExportStream(t *testing.T) { body, code := getWithAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream") assert.Equal(t, 200, code) - assert.Equal(t, 6, strings.Count(body, "\n")) + assert.Equal(t, 3, strings.Count(body, "\n")) assert.Equal(t, 2, strings.Count(body, "\"text\"")) t.Logf("%s", body) } @@ -487,7 +487,7 @@ func TestAdmin_ExportFile(t *testing.T) { assert.NoError(t, err) ungzBody, err := ioutil.ReadAll(ungzReader) assert.NoError(t, err) - assert.Equal(t, 6, strings.Count(string(ungzBody), "\n")) + assert.Equal(t, 3, strings.Count(string(ungzBody), "\n")) assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\"")) t.Logf("%s", string(ungzBody)) } diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index da860b25..83fa652a 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -34,7 +34,7 @@ func TestMigrator_Import(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} @@ -61,7 +61,7 @@ func TestMigrator_ImportForm(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) bodyBuf := &bytes.Buffer{} @@ -136,7 +136,7 @@ func TestMigrator_ImportRejected(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} @@ -157,7 +157,7 @@ func TestMigrator_ImportDouble(t *testing.T) { for i := 0; i < 1000; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } - r := strings.NewReader(strings.Join(recs, "\n")) // reader with 10k records + r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r) assert.Nil(t, err) @@ -184,7 +184,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) { for i := 0; i < 1000; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } - r := strings.NewReader(strings.Join(recs, "\n")) // reader with 10k records + r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r) require.Nil(t, err) @@ -205,7 +205,7 @@ func TestMigrator_Export(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) // import comments first @@ -234,7 +234,7 @@ func TestMigrator_Export(t *testing.T) { assert.NoError(t, err) ungzBody, err := ioutil.ReadAll(ungzReader) assert.NoError(t, err) - assert.Equal(t, 6, strings.Count(string(ungzBody), "\n")) + assert.Equal(t, 3, strings.Count(string(ungzBody), "\n")) assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\"")) t.Logf("%s", string(ungzBody)) @@ -248,7 +248,7 @@ func TestMigrator_Export(t *testing.T) { body, err := ioutil.ReadAll(resp.Body) assert.NoError(t, err) - assert.Equal(t, 6, strings.Count(string(body), "\n")) + assert.Equal(t, 3, strings.Count(string(body), "\n")) assert.Equal(t, 2, strings.Count(string(body), "\"text\"")) t.Logf("%s", string(body)) diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 56ab2f58..6a5d94c3 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -295,7 +295,7 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post // save users metas for _, um := range umetas { if um.Blocked.Status { - errs = multierror.Append(errs, s.SetBlock(siteID, um.ID, true, um.Blocked.Until.Sub(time.Now()))) + errs = multierror.Append(errs, s.SetBlock(siteID, um.ID, true, time.Until(um.Blocked.Until))) } if um.Verified { errs = multierror.Append(errs, s.SetVerified(siteID, um.ID, true))