diff --git a/README.md b/README.md
index d89447ef..df09b12c 100644
--- a/README.md
+++ b/README.md
@@ -769,7 +769,14 @@ _returned id should be appended to load image url on caller side_
* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
* `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
* `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
-* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completion.
+* `POST /api/v1/admin/remap?site=side-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n.
+From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on
+export/import chain so make backup first.
+ ```
+ http://oldsite.com* https://newsite.com*
+ http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
+ ```
+* `GET /api/v1/admin/wait?site=side-id` - wait for completion for any async migration ops (import or remap).
* `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
* `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
* `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.
diff --git a/backend/app/cmd/remap.go b/backend/app/cmd/remap.go
new file mode 100644
index 00000000..eda4fcc7
--- /dev/null
+++ b/backend/app/cmd/remap.go
@@ -0,0 +1,64 @@
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "time"
+
+ log "github.com/go-pkgz/lgr"
+ "github.com/pkg/errors"
+)
+
+// RemapCommand set of flags and command for change linkage between comments to
+// different urls based on given rules (input file)
+type RemapCommand struct {
+ Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+ InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
+ AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+ Timeout time.Duration `long:"timeout" default:"15m" description:"remap timeout"`
+ CommonOpts
+}
+
+func (rc *RemapCommand) Execute(args []string) error {
+ log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
+ resetEnv("SECRET", "ADMIN_PASSWD")
+
+ rulesReader, err := os.Open(rc.InputFile)
+ if err != nil {
+ return errors.Wrapf(err, "cant open file %s", rc.InputFile)
+ }
+
+ client := http.Client{}
+ ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout)
+ defer cancel()
+ remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site)
+ req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader)
+ if err != nil {
+ return errors.Wrapf(err, "can't make remap request for %s", remapURL)
+ }
+ req.SetBasicAuth("admin", rc.AdminPasswd)
+
+ resp, err := client.Do(req.WithContext(ctx))
+ if err != nil {
+ return errors.Wrapf(err, "request failed for %s", remapURL)
+ }
+ defer func() {
+ if err = resp.Body.Close(); err != nil {
+ log.Printf("[WARN] failed to close response, %s", err)
+ }
+ }()
+ if resp.StatusCode >= 300 {
+ return responseError(resp)
+ }
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return errors.Wrap(err, "can't get response")
+ }
+
+ log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
+ return nil
+}
diff --git a/backend/app/cmd/remap_test.go b/backend/app/cmd/remap_test.go
new file mode 100644
index 00000000..a04bdad7
--- /dev/null
+++ b/backend/app/cmd/remap_test.go
@@ -0,0 +1,36 @@
+package cmd
+
+import (
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/jessevdk/go-flags"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRemap_Execute(t *testing.T) {
+
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+ assert.Equal(t, "POST", r.Method)
+ assert.Equal(t, "remark", r.URL.Query().Get("site"))
+ body, err := ioutil.ReadAll(r.Body)
+ assert.Nil(t, err)
+ assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+ w.WriteHeader(202)
+ }))
+ defer ts.Close()
+
+ cmd := RemapCommand{}
+ cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+ p := flags.NewParser(&cmd, flags.Default)
+ _, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt", "--admin-passwd=secret"})
+ require.Nil(t, err)
+ err = cmd.Execute(nil)
+ assert.NoError(t, err)
+}
diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index cc7b47d7..999d9792 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -305,6 +305,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
DisqusImporter: &migrator.Disqus{DataStore: dataService},
WordPressImporter: &migrator.WordPress{DataStore: dataService},
NativeExporter: &migrator.Native{DataStore: dataService},
+ UrlMapperMaker: migrator.NewUrlMapper,
KeyStore: adminStore,
}
diff --git a/backend/app/cmd/testdata/remap_urls.txt b/backend/app/cmd/testdata/remap_urls.txt
new file mode 100644
index 00000000..dad90911
--- /dev/null
+++ b/backend/app/cmd/testdata/remap_urls.txt
@@ -0,0 +1,2 @@
+http://oldsite.com* https://newsite.com*
+http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
\ No newline at end of file
diff --git a/backend/app/main.go b/backend/app/main.go
index dcac34f9..1837a174 100644
--- a/backend/app/main.go
+++ b/backend/app/main.go
@@ -21,6 +21,7 @@ type Opts struct {
RestoreCmd cmd.RestoreCommand `command:"restore"`
AvatarCmd cmd.AvatarCommand `command:"avatar"`
CleanupCmd cmd.CleanupCommand `command:"cleanup"`
+ RemapCmd cmd.RemapCommand `command:"remap"`
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
diff --git a/backend/app/migrator/mapper.go b/backend/app/migrator/mapper.go
new file mode 100644
index 00000000..297fbec1
--- /dev/null
+++ b/backend/app/migrator/mapper.go
@@ -0,0 +1,74 @@
+package migrator
+
+import (
+ "errors"
+ "io"
+ "io/ioutil"
+ "strings"
+)
+
+// UrlMapper implements Mapper interface
+type UrlMapper struct {
+ rules map[string]string
+}
+
+// NewUrlMapper reads rules from given reader and returns initialised UrlMapper
+// if given rules are valid.
+func NewUrlMapper(reader io.Reader) (Mapper, error) {
+ u := &UrlMapper{}
+ if err := u.loadRules(reader); err != nil {
+ return u, err
+ }
+ return u, nil
+}
+
+// loadRules loads url-mapping rules from reader to mapper.
+// Rules must be a text consists of rows separated by \n.
+// Each row holds from-url and to-url separated by space.
+// If urls end with asterisk (*) it means try to match by prefix.
+// Example:
+// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
+// https://www.myblog.com/* https://myblog.com/*
+func (u *UrlMapper) loadRules(reader io.Reader) error {
+ data, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return err
+ }
+
+ rulesText := strings.TrimSpace(string(data))
+
+ u.rules = make(map[string]string)
+
+ for _, row := range strings.Split(rulesText, "\n") {
+ row = strings.TrimSpace(row)
+ urls := strings.Split(row, " ")
+ if len(urls) != 2 {
+ return errors.New("bad row " + row)
+ }
+
+ from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1])
+ u.rules[from] = to
+ }
+ return nil
+}
+
+// URL maps given url to another url according loaded url-rules.
+// If not matched returns given url.
+func (u *UrlMapper) URL(url string) string {
+ if newUrl, ok := u.rules[url]; ok {
+ return newUrl
+ }
+ // try to match by prefix
+ for oldUrl, newUrl := range u.rules {
+ if !strings.HasSuffix(oldUrl, "*") {
+ continue
+ }
+ oldUrl = strings.TrimSuffix(oldUrl, "*")
+ newUrl = strings.TrimSuffix(newUrl, "*")
+ if strings.HasPrefix(url, oldUrl) {
+ return newUrl + strings.TrimPrefix(url, oldUrl)
+ }
+ }
+ // search failed, return given url
+ return url
+}
diff --git a/backend/app/migrator/mapper_test.go b/backend/app/migrator/mapper_test.go
new file mode 100644
index 00000000..86c9ffab
--- /dev/null
+++ b/backend/app/migrator/mapper_test.go
@@ -0,0 +1,90 @@
+package migrator
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestUrlMapper_URL(t *testing.T) {
+ // want remap urls from https://radio-t.com to https://www.radio-t.com
+ // also map individual urls
+ rules := strings.NewReader(`
+https://radio-t.com* https://www.radio-t.com*
+https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/09/22/podcast-616/
+https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
+`)
+
+ mapper, err := NewUrlMapper(rules)
+ assert.NoError(t, err)
+
+ // if url not matched mapper should return given url
+ assert.Equal(t, "https://any.com/post/1/", mapper.URL("https://any.com/post/1/"))
+ assert.Equal(t, "https://radio-t.co", mapper.URL("https://radio-t.co"))
+ // check strict matching
+ assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22////podcast-616/"))
+ assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1"))
+ // check pattern matching (by prefix)
+ assert.Equal(t, "https://www.radio-t.com/p/post/123/", mapper.URL("https://radio-t.com/p/post/123/"))
+
+ // want remap from http to https
+ rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
+ mapper, err = NewUrlMapper(rules)
+ assert.NoError(t, err)
+ assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
+ assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+ assert.Equal(t, "https://anysite.com/", mapper.URL("https://anysite.com/")) // not exist
+
+ // want remap from http to https by pattern
+ rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
+ mapper, err = NewUrlMapper(rules)
+ assert.NoError(t, err)
+ assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
+ assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
+ assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+}
+
+func TestUrlMapper_New(t *testing.T) {
+ cases := []struct {
+ rules string
+ expectError bool
+ }{
+ // bad input, expect error
+ {
+ rules: "https://radio-t.com ",
+ expectError: true,
+ },
+ {
+ rules: "https://radio-t.com https://radio-t.com https://radio-t.com",
+ expectError: true,
+ },
+ {
+ rules: "https://radio-t.com https://radio-t.com\n https://radio-t.com",
+ expectError: true,
+ },
+ {
+ rules: "https://radio-t.com \n https://radio-t.com https://radio-t.com",
+ expectError: true,
+ },
+
+ // valid input, no error
+ {
+ rules: "https://radio-t.com* https://www.radio-t.com*",
+ },
+ {
+ rules: "https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/",
+ },
+ {
+ rules: "https://any.com/p/111 https://any.com/p/222 \n https://any.com/p/333 https://any.com/p/222 \n",
+ },
+ }
+ for _, c := range cases {
+ _, err := NewUrlMapper(strings.NewReader(c.rules))
+ if c.expectError {
+ assert.Error(t, err)
+ } else {
+ assert.Nil(t, err)
+ }
+ }
+}
diff --git a/backend/app/migrator/migrator.go b/backend/app/migrator/migrator.go
index 93ba4b53..0d7dd4d7 100644
--- a/backend/app/migrator/migrator.go
+++ b/backend/app/migrator/migrator.go
@@ -24,6 +24,16 @@ type Exporter interface {
Export(w io.Writer, siteID string) (int, error)
}
+// Mapper defines interface to convert data in import procedure
+type Mapper interface {
+ URL(url string) string
+}
+
+// MapperMaker defines function that reads rules from reader and
+// returns new Mapper with loaded rules. If rules are not valid
+// it returns error.
+type MapperMaker func(reader io.Reader) (Mapper, error)
+
// Store defines minimal interface needed to export and import comments
type Store interface {
Create(comment store.Comment) (commentID string, err error)
diff --git a/backend/app/migrator/native.go b/backend/app/migrator/native.go
index 6084dd24..9ab7d1e8 100644
--- a/backend/app/migrator/native.go
+++ b/backend/app/migrator/native.go
@@ -87,9 +87,49 @@ func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
return nil
}
+// WithMapper wraps reader with url-mapper.
+func WithMapper(reader io.Reader, mapper Mapper) io.Reader {
+ r, w := io.Pipe()
+ go func() {
+ var err error
+ defer func() {
+ log.Printf("[DEBUG] finish write to pipe with %+v", err)
+ if e := w.Close(); e != nil {
+ log.Printf("[WARN] failed close pipe writer with %+v", e)
+ }
+ }()
+
+ // decode from reader and encode to pipe writer
+ dec, enc := json.NewDecoder(reader), json.NewEncoder(w)
+
+ m := meta{}
+ if err = dec.Decode(&m); err != nil {
+ return
+ }
+ for i := range m.Posts {
+ m.Posts[i].URL = mapper.URL(m.Posts[i].URL)
+ }
+ if err = enc.Encode(m); err != nil {
+ return
+ }
+
+ for {
+ comment := store.Comment{}
+ if err = dec.Decode(&comment); err != nil {
+ return
+ }
+ comment.Locator.URL = mapper.URL(comment.Locator.URL)
+ if err = enc.Encode(comment); err != nil {
+ return
+ }
+ }
+ }()
+
+ return r
+}
+
// Import comments from json strings produced by Remark.Export
func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
-
m := meta{}
dec := json.NewDecoder(reader)
if err = dec.Decode(&m); err != nil {
diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go
index 280d7f63..dd243f0e 100644
--- a/backend/app/migrator/native_test.go
+++ b/backend/app/migrator/native_test.go
@@ -101,6 +101,45 @@ func TestNative_Import(t *testing.T) {
assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
}
+func TestNative_ImportWithMapper(t *testing.T) {
+ defer os.Remove(testDb)
+
+ // want to remap comments to https://rdt.c
+ rules := `https://radio-t.com* https://rdt.c*`
+ mapper, err := NewUrlMapper(strings.NewReader(rules))
+ assert.NoError(t, err)
+
+ inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+ {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+ {"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+ mappedReader := WithMapper(strings.NewReader(inp), mapper)
+
+ b := prep(t) // write some recs, they will be deleted
+ b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+ r := Native{DataStore: b}
+ size, err := r.Import(mappedReader, "radio-t")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, size)
+
+ comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(comments))
+ assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
+ assert.Equal(t, "1234", comments[0].ParentID)
+ assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+ assert.Equal(t, "https://rdt.c/2", comments[0].Locator.URL)
+
+ assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
+ assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+ assert.Equal(t, "https://rdt.c", comments[1].Locator.URL)
+
+ assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
+ assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
+
+ assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
+ assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
+}
+
func TestNative_ImportWrongVersion(t *testing.T) {
inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go
index 075a3f6c..8779bf80 100644
--- a/backend/app/rest/api/migrator.go
+++ b/backend/app/rest/api/migrator.go
@@ -28,6 +28,7 @@ type Migrator struct {
DisqusImporter migrator.Importer
WordPressImporter migrator.Importer
NativeExporter migrator.Exporter
+ UrlMapperMaker migrator.MapperMaker
KeyStore KeyStore
busy map[string]bool
@@ -98,7 +99,9 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, R.JSON{"status": "import request accepted"})
}
-func (m *Migrator) importWaitCtrl(w http.ResponseWriter, r *http.Request) {
+// GET /wait?site=site-id
+// waits for migration operation (import or remap)
+func (m *Migrator) waitCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
timeOut := time.Minute * 15
if v := r.URL.Query().Get("timeout"); v != "" {
@@ -152,6 +155,62 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
}
}
+// POST /remap?site=site-id
+// remap urls in comments based on given rules (oldUrl newUrl)
+func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
+ siteID := r.URL.Query().Get("site")
+
+ // create new url-mapper from given rules in body
+ mapper, err := m.UrlMapperMaker(r.Body)
+ if err != nil {
+ rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
+ return
+ }
+ defer r.Body.Close()
+
+ // start remap procedure with mapper
+ go func() {
+ m.setBusy(siteID, true)
+ defer m.setBusy(siteID, false)
+
+ // do export
+ fh, err := ioutil.TempFile("", "remark42_convert")
+ if err != nil {
+ log.Printf("[WARN] failed to make temp file %+v", err)
+ return
+ }
+ defer func() {
+ if err := os.Remove(fh.Name()); err != nil {
+ log.Printf("[WARN] failed to remove temp file %+v", err)
+ }
+ }()
+ log.Printf("[DEBUG] start export for site=%s", siteID)
+ if _, err := m.NativeExporter.Export(fh, siteID); err != nil {
+ log.Printf("[WARN] export failed with %+v", err)
+ return
+ }
+
+ if _, err = fh.Seek(0, 0); err != nil {
+ log.Printf("[WARN] failed to seek file %+v", err)
+ return
+ }
+
+ log.Printf("[DEBUG] start import for site=%s", siteID)
+ mappedReader := migrator.WithMapper(fh, mapper)
+ size, err := m.NativeImporter.Import(mappedReader, siteID)
+ if err != nil {
+ log.Printf("[WARN] import failed with %+v", err)
+ return
+ }
+
+ m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
+ log.Printf("[DEBUG] convert request completed. site=%s, comments=%d", siteID, size)
+ }()
+
+ render.Status(r, http.StatusAccepted)
+ render.JSON(w, r, R.JSON{"status": "convert request accepted"})
+}
+
// runImport reads from tmpfile and import for given siteID and provider
func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
m.setBusy(siteID, true)
diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go
index 51b38804..c667cb87 100644
--- a/backend/app/rest/api/migrator_test.go
+++ b/backend/app/rest/api/migrator_test.go
@@ -3,6 +3,7 @@ package api
import (
"bytes"
"compress/gzip"
+ "encoding/json"
"fmt"
"io"
"io/ioutil"
@@ -15,6 +16,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/store"
+ "github.com/umputun/remark/backend/app/store/service"
)
func TestMigrator_Import(t *testing.T) {
@@ -43,7 +47,7 @@ func TestMigrator_Import(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
- waitForImportCompletion(t, ts)
+ waitForMigrationCompletion(t, ts)
}
func TestMigrator_ImportForm(t *testing.T) {
@@ -77,7 +81,7 @@ func TestMigrator_ImportForm(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
- waitForImportCompletion(t, ts)
+ waitForMigrationCompletion(t, ts)
}
func TestMigrator_ImportFromWP(t *testing.T) {
@@ -99,7 +103,7 @@ func TestMigrator_ImportFromWP(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
- waitForImportCompletion(t, ts)
+ waitForMigrationCompletion(t, ts)
}
func TestMigrator_ImportRejected(t *testing.T) {
@@ -153,7 +157,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
resp, err = client.Do(req)
assert.Nil(t, err)
assert.Equal(t, http.StatusConflict, resp.StatusCode)
- waitForImportCompletion(t, ts)
+ waitForMigrationCompletion(t, ts)
}
func TestMigrator_ImportWaitExpired(t *testing.T) {
@@ -179,7 +183,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
client = &http.Client{Timeout: 10 * time.Second}
- req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=remark42&timeout=100ms", nil)
+ req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=100ms", nil)
require.NoError(t, err)
req.SetBasicAuth("admin", "password")
assert.NoError(t, err)
@@ -187,7 +191,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
- waitForImportCompletion(t, ts)
+ waitForMigrationCompletion(t, ts)
}
func TestMigrator_Export(t *testing.T) {
@@ -211,7 +215,7 @@ func TestMigrator_Export(t *testing.T) {
resp, err := client.Do(req)
require.Nil(t, err)
require.Equal(t, http.StatusAccepted, resp.StatusCode)
- waitForImportCompletion(t, ts)
+ waitForMigrationCompletion(t, ts)
// check file mode
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", nil)
@@ -252,9 +256,107 @@ func TestMigrator_Export(t *testing.T) {
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
-func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
+func TestMigrator_Remap(t *testing.T) {
+ ts, srv, teardown := startupT(t)
+ defer teardown()
+
+ // create 2 comments in https://remark42.com/demo/
+ c1 := store.Comment{Text: "first comment", Timestamp: time.Now(),
+ Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u1"}}
+ _, err := srv.DataService.Create(c1)
+ require.NoError(t, err)
+ c2 := store.Comment{Text: "second comment", Timestamp: time.Now(),
+ Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u2"}}
+ _, err = srv.DataService.Create(c2)
+ require.NoError(t, err)
+
+ // create 1 comment in https://remark42.com/demo-another/
+ c3 := store.Comment{Text: "third comment", Timestamp: time.Now(),
+ Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo-another/"}, User: store.User{ID: "u3"}}
+ _, err = srv.DataService.Create(c3)
+ require.NoError(t, err)
+
+ // set url https://remark42.com/demo-another/ to be readonly
+ err = srv.DataService.SetMetas("remark42", []service.UserMetaData{}, []service.PostMetaData{{
+ URL: "https://remark42.com/demo-another/",
+ ReadOnly: true,
+ }})
+ require.NoError(t, err)
+
+ // check that comments created as expected
+ res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+ require.Equal(t, 200, code)
+ comments := commentsWithInfo{}
+ err = json.Unmarshal([]byte(res), &comments)
+ require.Nil(t, err)
+ require.Equal(t, 2, comments.Info.Count)
+ require.False(t, comments.Info.ReadOnly)
+
+ res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+ require.Equal(t, 200, code)
+ comments = commentsWithInfo{}
+ err = json.Unmarshal([]byte(res), &comments)
+ require.Nil(t, err)
+ require.Equal(t, 1, comments.Info.Count)
+ require.True(t, comments.Info.ReadOnly)
+
+ // we want remap urls to another domain - www.remark42.com
+ rules := "https://remark42.com/* https://www.remark42.com/*"
+ resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
+ require.Nil(t, err)
+ require.Equal(t, http.StatusAccepted, resp.StatusCode)
+ waitForMigrationCompletion(t, ts)
+
+ // after remap finished we should find comments from new urls
+ res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo/")
+ require.Equal(t, 200, code)
+ comments = commentsWithInfo{}
+ err = json.Unmarshal([]byte(res), &comments)
+ require.Nil(t, err)
+ require.Equal(t, 2, comments.Info.Count)
+ require.False(t, comments.Info.ReadOnly)
+
+ res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
+ require.Equal(t, 200, code)
+ comments = commentsWithInfo{}
+ err = json.Unmarshal([]byte(res), &comments)
+ require.Nil(t, err)
+ require.Equal(t, 1, comments.Info.Count)
+ require.True(t, comments.Info.ReadOnly)
+
+ // should find nothing from previous url
+ res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+ require.Equal(t, 200, code)
+ comments = commentsWithInfo{}
+ err = json.Unmarshal([]byte(res), &comments)
+ require.Nil(t, err)
+ require.Equal(t, 0, comments.Info.Count)
+
+ res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+ require.Equal(t, 200, code)
+ comments = commentsWithInfo{}
+ err = json.Unmarshal([]byte(res), &comments)
+ require.Nil(t, err)
+ require.Equal(t, 0, comments.Info.Count)
+}
+
+func TestMigrator_RemapReject(t *testing.T) {
+ ts, _, teardown := startupT(t)
+ defer teardown()
+
+ // without admin credentials
+ client := &http.Client{Timeout: 1 * time.Second}
+ rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`)
+ req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules)
+ require.Nil(t, err)
+ resp, err := client.Do(req)
+ require.Nil(t, err)
+ require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
+func waitForMigrationCompletion(t *testing.T, ts *httptest.Server) {
client := &http.Client{Timeout: 10 * time.Second}
- req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=remark42", nil)
+ req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42", nil)
require.NoError(t, err)
req.SetBasicAuth("admin", "password")
assert.NoError(t, err)
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index 9ea621e7..9d139945 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -285,7 +285,8 @@ func (s *Rest) routes() chi.Router {
radmin.Get("/export", s.adminRest.migrator.exportCtrl)
radmin.Post("/import", s.adminRest.migrator.importCtrl)
radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl)
- radmin.Get("/import/wait", s.adminRest.migrator.importWaitCtrl)
+ radmin.Post("/remap", s.adminRest.migrator.remapCtrl)
+ radmin.Get("/wait", s.adminRest.migrator.waitCtrl)
})
// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index 22e83122..4fecd87e 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -330,7 +330,8 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
WordPressImporter: &migrator.WordPress{DataStore: dataStore},
NativeImporter: &migrator.Native{DataStore: dataStore},
NativeExporter: &migrator.Native{DataStore: dataStore},
- Cache: &cache.Nop{},
+ UrlMapperMaker: migrator.NewUrlMapper,
+ Cache: memCache,
KeyStore: astore,
},
Streamer: &Streamer{
diff --git a/backend/go.mod b/backend/go.mod
index 6aae1eed..4b6e33b6 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -56,5 +56,6 @@ require (
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
golang.org/x/image v0.0.0-20190823064033-3a9bac650e44
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
+ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/russross/blackfriday.v2 v2.0.1
)