From 4dc123dbe84f4f248864bcdbbd6cc2b3a4dafe11 Mon Sep 17 00:00:00 2001 From: Anton Kosourov Date: Mon, 30 Sep 2019 21:12:42 +0300 Subject: [PATCH] Change url in comments, decorate approach (#431) * WIP: url mapper, wrapped reader approach * create url mapper on start * add pattern matching in mapper * check pattern matching in test * change site-id from radio-t to remark42 in tests * create new url mapper on demand, based on given func, union strict and prefix rules * rename convert to remap * add import with mapper test * rename mapper func to UrlMapperMaker, create comments in test via data service * move /import/wait to /wait ctrl * add remap cmd * fix url naming --- README.md | 9 +- backend/app/cmd/remap.go | 64 +++++++++++++ backend/app/cmd/remap_test.go | 36 +++++++ backend/app/cmd/server.go | 1 + backend/app/cmd/testdata/remap_urls.txt | 2 + backend/app/main.go | 1 + backend/app/migrator/mapper.go | 74 +++++++++++++++ backend/app/migrator/mapper_test.go | 90 ++++++++++++++++++ backend/app/migrator/migrator.go | 10 ++ backend/app/migrator/native.go | 42 ++++++++- backend/app/migrator/native_test.go | 39 ++++++++ backend/app/rest/api/migrator.go | 61 +++++++++++- backend/app/rest/api/migrator_test.go | 120 ++++++++++++++++++++++-- backend/app/rest/api/rest.go | 3 +- backend/app/rest/api/rest_test.go | 3 +- backend/go.mod | 1 + 16 files changed, 542 insertions(+), 14 deletions(-) create mode 100644 backend/app/cmd/remap.go create mode 100644 backend/app/cmd/remap_test.go create mode 100644 backend/app/cmd/testdata/remap_urls.txt create mode 100644 backend/app/migrator/mapper.go create mode 100644 backend/app/migrator/mapper_test.go diff --git a/README.md b/README.md index d89447ef..df09b12c 100644 --- a/README.md +++ b/README.md @@ -769,7 +769,14 @@ _returned id should be appended to load image url on caller side_ * `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file. * `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body. * `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form. -* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completion. +* `POST /api/v1/admin/remap?site=side-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n. +From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on +export/import chain so make backup first. + ``` + http://oldsite.com* https://newsite.com* + http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1 + ``` +* `GET /api/v1/admin/wait?site=side-id` - wait for completion for any async migration ops (import or remap). * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment. * `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info. * `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments. diff --git a/backend/app/cmd/remap.go b/backend/app/cmd/remap.go new file mode 100644 index 00000000..eda4fcc7 --- /dev/null +++ b/backend/app/cmd/remap.go @@ -0,0 +1,64 @@ +package cmd + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "os" + "time" + + log "github.com/go-pkgz/lgr" + "github.com/pkg/errors" +) + +// RemapCommand set of flags and command for change linkage between comments to +// different urls based on given rules (input file) +type RemapCommand struct { + Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` + InputFile string `short:"f" long:"file" description:"input file name" required:"true"` + AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"` + Timeout time.Duration `long:"timeout" default:"15m" description:"remap timeout"` + CommonOpts +} + +func (rc *RemapCommand) Execute(args []string) error { + log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile) + resetEnv("SECRET", "ADMIN_PASSWD") + + rulesReader, err := os.Open(rc.InputFile) + if err != nil { + return errors.Wrapf(err, "cant open file %s", rc.InputFile) + } + + client := http.Client{} + ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout) + defer cancel() + remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site) + req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader) + if err != nil { + return errors.Wrapf(err, "can't make remap request for %s", remapURL) + } + req.SetBasicAuth("admin", rc.AdminPasswd) + + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrapf(err, "request failed for %s", remapURL) + } + defer func() { + if err = resp.Body.Close(); err != nil { + log.Printf("[WARN] failed to close response, %s", err) + } + }() + if resp.StatusCode >= 300 { + return responseError(resp) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "can't get response") + } + + log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body)) + return nil +} diff --git a/backend/app/cmd/remap_test.go b/backend/app/cmd/remap_test.go new file mode 100644 index 00000000..a04bdad7 --- /dev/null +++ b/backend/app/cmd/remap_test.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRemap_Execute(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/remap") + assert.Equal(t, "POST", r.Method) + assert.Equal(t, "remark", r.URL.Query().Get("site")) + body, err := ioutil.ReadAll(r.Body) + assert.Nil(t, err) + assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body)) + + w.WriteHeader(202) + })) + defer ts.Close() + + cmd := RemapCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) + + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt", "--admin-passwd=secret"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) +} diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index cc7b47d7..999d9792 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -305,6 +305,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { DisqusImporter: &migrator.Disqus{DataStore: dataService}, WordPressImporter: &migrator.WordPress{DataStore: dataService}, NativeExporter: &migrator.Native{DataStore: dataService}, + UrlMapperMaker: migrator.NewUrlMapper, KeyStore: adminStore, } diff --git a/backend/app/cmd/testdata/remap_urls.txt b/backend/app/cmd/testdata/remap_urls.txt new file mode 100644 index 00000000..dad90911 --- /dev/null +++ b/backend/app/cmd/testdata/remap_urls.txt @@ -0,0 +1,2 @@ +http://oldsite.com* https://newsite.com* +http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1 \ No newline at end of file diff --git a/backend/app/main.go b/backend/app/main.go index dcac34f9..1837a174 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -21,6 +21,7 @@ type Opts struct { RestoreCmd cmd.RestoreCommand `command:"restore"` AvatarCmd cmd.AvatarCommand `command:"avatar"` CleanupCmd cmd.CleanupCommand `command:"cleanup"` + RemapCmd cmd.RemapCommand `command:"remap"` RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` diff --git a/backend/app/migrator/mapper.go b/backend/app/migrator/mapper.go new file mode 100644 index 00000000..297fbec1 --- /dev/null +++ b/backend/app/migrator/mapper.go @@ -0,0 +1,74 @@ +package migrator + +import ( + "errors" + "io" + "io/ioutil" + "strings" +) + +// UrlMapper implements Mapper interface +type UrlMapper struct { + rules map[string]string +} + +// NewUrlMapper reads rules from given reader and returns initialised UrlMapper +// if given rules are valid. +func NewUrlMapper(reader io.Reader) (Mapper, error) { + u := &UrlMapper{} + if err := u.loadRules(reader); err != nil { + return u, err + } + return u, nil +} + +// loadRules loads url-mapping rules from reader to mapper. +// Rules must be a text consists of rows separated by \n. +// Each row holds from-url and to-url separated by space. +// If urls end with asterisk (*) it means try to match by prefix. +// Example: +// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/ +// https://www.myblog.com/* https://myblog.com/* +func (u *UrlMapper) loadRules(reader io.Reader) error { + data, err := ioutil.ReadAll(reader) + if err != nil { + return err + } + + rulesText := strings.TrimSpace(string(data)) + + u.rules = make(map[string]string) + + for _, row := range strings.Split(rulesText, "\n") { + row = strings.TrimSpace(row) + urls := strings.Split(row, " ") + if len(urls) != 2 { + return errors.New("bad row " + row) + } + + from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1]) + u.rules[from] = to + } + return nil +} + +// URL maps given url to another url according loaded url-rules. +// If not matched returns given url. +func (u *UrlMapper) URL(url string) string { + if newUrl, ok := u.rules[url]; ok { + return newUrl + } + // try to match by prefix + for oldUrl, newUrl := range u.rules { + if !strings.HasSuffix(oldUrl, "*") { + continue + } + oldUrl = strings.TrimSuffix(oldUrl, "*") + newUrl = strings.TrimSuffix(newUrl, "*") + if strings.HasPrefix(url, oldUrl) { + return newUrl + strings.TrimPrefix(url, oldUrl) + } + } + // search failed, return given url + return url +} diff --git a/backend/app/migrator/mapper_test.go b/backend/app/migrator/mapper_test.go new file mode 100644 index 00000000..86c9ffab --- /dev/null +++ b/backend/app/migrator/mapper_test.go @@ -0,0 +1,90 @@ +package migrator + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUrlMapper_URL(t *testing.T) { + // want remap urls from https://radio-t.com to https://www.radio-t.com + // also map individual urls + rules := strings.NewReader(` +https://radio-t.com* https://www.radio-t.com* +https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/09/22/podcast-616/ +https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/ +`) + + mapper, err := NewUrlMapper(rules) + assert.NoError(t, err) + + // if url not matched mapper should return given url + assert.Equal(t, "https://any.com/post/1/", mapper.URL("https://any.com/post/1/")) + assert.Equal(t, "https://radio-t.co", mapper.URL("https://radio-t.co")) + // check strict matching + assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22////podcast-616/")) + assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1")) + // check pattern matching (by prefix) + assert.Equal(t, "https://www.radio-t.com/p/post/123/", mapper.URL("https://radio-t.com/p/post/123/")) + + // want remap from http to https + rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`) + mapper, err = NewUrlMapper(rules) + assert.NoError(t, err) + assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123")) + assert.Equal(t, "https://notexist", mapper.URL("https://notexist")) + assert.Equal(t, "https://anysite.com/", mapper.URL("https://anysite.com/")) // not exist + + // want remap from http to https by pattern + rules = strings.NewReader(`http://anysite.com* https://anysite.com*`) + mapper, err = NewUrlMapper(rules) + assert.NoError(t, err) + assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1")) + assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/")) + assert.Equal(t, "https://notexist", mapper.URL("https://notexist")) +} + +func TestUrlMapper_New(t *testing.T) { + cases := []struct { + rules string + expectError bool + }{ + // bad input, expect error + { + rules: "https://radio-t.com ", + expectError: true, + }, + { + rules: "https://radio-t.com https://radio-t.com https://radio-t.com", + expectError: true, + }, + { + rules: "https://radio-t.com https://radio-t.com\n https://radio-t.com", + expectError: true, + }, + { + rules: "https://radio-t.com \n https://radio-t.com https://radio-t.com", + expectError: true, + }, + + // valid input, no error + { + rules: "https://radio-t.com* https://www.radio-t.com*", + }, + { + rules: "https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/", + }, + { + rules: "https://any.com/p/111 https://any.com/p/222 \n https://any.com/p/333 https://any.com/p/222 \n", + }, + } + for _, c := range cases { + _, err := NewUrlMapper(strings.NewReader(c.rules)) + if c.expectError { + assert.Error(t, err) + } else { + assert.Nil(t, err) + } + } +} diff --git a/backend/app/migrator/migrator.go b/backend/app/migrator/migrator.go index 93ba4b53..0d7dd4d7 100644 --- a/backend/app/migrator/migrator.go +++ b/backend/app/migrator/migrator.go @@ -24,6 +24,16 @@ type Exporter interface { Export(w io.Writer, siteID string) (int, error) } +// Mapper defines interface to convert data in import procedure +type Mapper interface { + URL(url string) string +} + +// MapperMaker defines function that reads rules from reader and +// returns new Mapper with loaded rules. If rules are not valid +// it returns error. +type MapperMaker func(reader io.Reader) (Mapper, error) + // Store defines minimal interface needed to export and import comments type Store interface { Create(comment store.Comment) (commentID string, err error) diff --git a/backend/app/migrator/native.go b/backend/app/migrator/native.go index 6084dd24..9ab7d1e8 100644 --- a/backend/app/migrator/native.go +++ b/backend/app/migrator/native.go @@ -87,9 +87,49 @@ func (n *Native) exportMeta(siteID string, w io.Writer) (err error) { return nil } +// WithMapper wraps reader with url-mapper. +func WithMapper(reader io.Reader, mapper Mapper) io.Reader { + r, w := io.Pipe() + go func() { + var err error + defer func() { + log.Printf("[DEBUG] finish write to pipe with %+v", err) + if e := w.Close(); e != nil { + log.Printf("[WARN] failed close pipe writer with %+v", e) + } + }() + + // decode from reader and encode to pipe writer + dec, enc := json.NewDecoder(reader), json.NewEncoder(w) + + m := meta{} + if err = dec.Decode(&m); err != nil { + return + } + for i := range m.Posts { + m.Posts[i].URL = mapper.URL(m.Posts[i].URL) + } + if err = enc.Encode(m); err != nil { + return + } + + for { + comment := store.Comment{} + if err = dec.Decode(&comment); err != nil { + return + } + comment.Locator.URL = mapper.URL(comment.Locator.URL) + if err = enc.Encode(comment); err != nil { + return + } + } + }() + + return r +} + // Import comments from json strings produced by Remark.Export func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { - m := meta{} dec := json.NewDecoder(reader) if err = dec.Decode(&m); err != nil { diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go index 280d7f63..dd243f0e 100644 --- a/backend/app/migrator/native_test.go +++ b/backend/app/migrator/native_test.go @@ -101,6 +101,45 @@ func TestNative_Import(t *testing.T) { assert.Equal(t, false, b.IsVerified("radio-t", "user2")) } +func TestNative_ImportWithMapper(t *testing.T) { + defer os.Remove(testDb) + + // want to remap comments to https://rdt.c + rules := `https://radio-t.com* https://rdt.c*` + mapper, err := NewUrlMapper(strings.NewReader(rules)) + assert.NoError(t, err) + + inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]} + {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"} + {"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + mappedReader := WithMapper(strings.NewReader(inp), mapper) + + b := prep(t) // write some recs, they will be deleted + b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "") + r := Native{DataStore: b} + size, err := r.Import(mappedReader, "radio-t") + assert.Nil(t, err) + assert.Equal(t, 2, size) + + comments, err := b.Last("radio-t", 10, time.Time{}, store.User{}) + assert.Nil(t, err) + assert.Equal(t, 2, len(comments)) + assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID) + assert.Equal(t, "1234", comments[0].ParentID) + assert.Equal(t, false, b.IsReadOnly(comments[0].Locator)) + assert.Equal(t, "https://rdt.c/2", comments[0].Locator.URL) + + assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID) + assert.Equal(t, true, b.IsReadOnly(comments[1].Locator)) + assert.Equal(t, "https://rdt.c", comments[1].Locator.URL) + + assert.Equal(t, false, b.IsBlocked("radio-t", "user1")) + assert.Equal(t, true, b.IsVerified("radio-t", "user1")) + + assert.Equal(t, true, b.IsBlocked("radio-t", "user2")) + assert.Equal(t, false, b.IsVerified("radio-t", "user2")) +} + func TestNative_ImportWrongVersion(t *testing.T) { inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"} diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 075a3f6c..8779bf80 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -28,6 +28,7 @@ type Migrator struct { DisqusImporter migrator.Importer WordPressImporter migrator.Importer NativeExporter migrator.Exporter + UrlMapperMaker migrator.MapperMaker KeyStore KeyStore busy map[string]bool @@ -98,7 +99,9 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, R.JSON{"status": "import request accepted"}) } -func (m *Migrator) importWaitCtrl(w http.ResponseWriter, r *http.Request) { +// GET /wait?site=site-id +// waits for migration operation (import or remap) +func (m *Migrator) waitCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") timeOut := time.Minute * 15 if v := r.URL.Query().Get("timeout"); v != "" { @@ -152,6 +155,62 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { } } +// POST /remap?site=site-id +// remap urls in comments based on given rules (oldUrl newUrl) +func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site") + + // create new url-mapper from given rules in body + mapper, err := m.UrlMapperMaker(r.Body) + if err != nil { + rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode) + return + } + defer r.Body.Close() + + // start remap procedure with mapper + go func() { + m.setBusy(siteID, true) + defer m.setBusy(siteID, false) + + // do export + fh, err := ioutil.TempFile("", "remark42_convert") + if err != nil { + log.Printf("[WARN] failed to make temp file %+v", err) + return + } + defer func() { + if err := os.Remove(fh.Name()); err != nil { + log.Printf("[WARN] failed to remove temp file %+v", err) + } + }() + log.Printf("[DEBUG] start export for site=%s", siteID) + if _, err := m.NativeExporter.Export(fh, siteID); err != nil { + log.Printf("[WARN] export failed with %+v", err) + return + } + + if _, err = fh.Seek(0, 0); err != nil { + log.Printf("[WARN] failed to seek file %+v", err) + return + } + + log.Printf("[DEBUG] start import for site=%s", siteID) + mappedReader := migrator.WithMapper(fh, mapper) + size, err := m.NativeImporter.Import(mappedReader, siteID) + if err != nil { + log.Printf("[WARN] import failed with %+v", err) + return + } + + m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID)) + log.Printf("[DEBUG] convert request completed. site=%s, comments=%d", siteID, size) + }() + + render.Status(r, http.StatusAccepted) + render.JSON(w, r, R.JSON{"status": "convert request accepted"}) +} + // runImport reads from tmpfile and import for given siteID and provider func (m *Migrator) runImport(siteID string, provider string, tmpfile string) { m.setBusy(siteID, true) diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 51b38804..c667cb87 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -3,6 +3,7 @@ package api import ( "bytes" "compress/gzip" + "encoding/json" "fmt" "io" "io/ioutil" @@ -15,6 +16,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/service" ) func TestMigrator_Import(t *testing.T) { @@ -43,7 +47,7 @@ func TestMigrator_Import(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) - waitForImportCompletion(t, ts) + waitForMigrationCompletion(t, ts) } func TestMigrator_ImportForm(t *testing.T) { @@ -77,7 +81,7 @@ func TestMigrator_ImportForm(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) - waitForImportCompletion(t, ts) + waitForMigrationCompletion(t, ts) } func TestMigrator_ImportFromWP(t *testing.T) { @@ -99,7 +103,7 @@ func TestMigrator_ImportFromWP(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) - waitForImportCompletion(t, ts) + waitForMigrationCompletion(t, ts) } func TestMigrator_ImportRejected(t *testing.T) { @@ -153,7 +157,7 @@ func TestMigrator_ImportDouble(t *testing.T) { resp, err = client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusConflict, resp.StatusCode) - waitForImportCompletion(t, ts) + waitForMigrationCompletion(t, ts) } func TestMigrator_ImportWaitExpired(t *testing.T) { @@ -179,7 +183,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) { assert.Equal(t, http.StatusAccepted, resp.StatusCode) client = &http.Client{Timeout: 10 * time.Second} - req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=remark42&timeout=100ms", nil) + req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=100ms", nil) require.NoError(t, err) req.SetBasicAuth("admin", "password") assert.NoError(t, err) @@ -187,7 +191,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) { assert.NoError(t, err) assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode) - waitForImportCompletion(t, ts) + waitForMigrationCompletion(t, ts) } func TestMigrator_Export(t *testing.T) { @@ -211,7 +215,7 @@ func TestMigrator_Export(t *testing.T) { resp, err := client.Do(req) require.Nil(t, err) require.Equal(t, http.StatusAccepted, resp.StatusCode) - waitForImportCompletion(t, ts) + waitForMigrationCompletion(t, ts) // check file mode req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", nil) @@ -252,9 +256,107 @@ func TestMigrator_Export(t *testing.T) { require.Equal(t, http.StatusUnauthorized, resp.StatusCode) } -func waitForImportCompletion(t *testing.T, ts *httptest.Server) { +func TestMigrator_Remap(t *testing.T) { + ts, srv, teardown := startupT(t) + defer teardown() + + // create 2 comments in https://remark42.com/demo/ + c1 := store.Comment{Text: "first comment", Timestamp: time.Now(), + Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u1"}} + _, err := srv.DataService.Create(c1) + require.NoError(t, err) + c2 := store.Comment{Text: "second comment", Timestamp: time.Now(), + Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u2"}} + _, err = srv.DataService.Create(c2) + require.NoError(t, err) + + // create 1 comment in https://remark42.com/demo-another/ + c3 := store.Comment{Text: "third comment", Timestamp: time.Now(), + Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo-another/"}, User: store.User{ID: "u3"}} + _, err = srv.DataService.Create(c3) + require.NoError(t, err) + + // set url https://remark42.com/demo-another/ to be readonly + err = srv.DataService.SetMetas("remark42", []service.UserMetaData{}, []service.PostMetaData{{ + URL: "https://remark42.com/demo-another/", + ReadOnly: true, + }}) + require.NoError(t, err) + + // check that comments created as expected + res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/") + require.Equal(t, 200, code) + comments := commentsWithInfo{} + err = json.Unmarshal([]byte(res), &comments) + require.Nil(t, err) + require.Equal(t, 2, comments.Info.Count) + require.False(t, comments.Info.ReadOnly) + + res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/") + require.Equal(t, 200, code) + comments = commentsWithInfo{} + err = json.Unmarshal([]byte(res), &comments) + require.Nil(t, err) + require.Equal(t, 1, comments.Info.Count) + require.True(t, comments.Info.ReadOnly) + + // we want remap urls to another domain - www.remark42.com + rules := "https://remark42.com/* https://www.remark42.com/*" + resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin + require.Nil(t, err) + require.Equal(t, http.StatusAccepted, resp.StatusCode) + waitForMigrationCompletion(t, ts) + + // after remap finished we should find comments from new urls + res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo/") + require.Equal(t, 200, code) + comments = commentsWithInfo{} + err = json.Unmarshal([]byte(res), &comments) + require.Nil(t, err) + require.Equal(t, 2, comments.Info.Count) + require.False(t, comments.Info.ReadOnly) + + res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/") + require.Equal(t, 200, code) + comments = commentsWithInfo{} + err = json.Unmarshal([]byte(res), &comments) + require.Nil(t, err) + require.Equal(t, 1, comments.Info.Count) + require.True(t, comments.Info.ReadOnly) + + // should find nothing from previous url + res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/") + require.Equal(t, 200, code) + comments = commentsWithInfo{} + err = json.Unmarshal([]byte(res), &comments) + require.Nil(t, err) + require.Equal(t, 0, comments.Info.Count) + + res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/") + require.Equal(t, 200, code) + comments = commentsWithInfo{} + err = json.Unmarshal([]byte(res), &comments) + require.Nil(t, err) + require.Equal(t, 0, comments.Info.Count) +} + +func TestMigrator_RemapReject(t *testing.T) { + ts, _, teardown := startupT(t) + defer teardown() + + // without admin credentials + client := &http.Client{Timeout: 1 * time.Second} + rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`) + req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules) + require.Nil(t, err) + resp, err := client.Do(req) + require.Nil(t, err) + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) +} + +func waitForMigrationCompletion(t *testing.T, ts *httptest.Server) { client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=remark42", nil) + req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42", nil) require.NoError(t, err) req.SetBasicAuth("admin", "password") assert.NoError(t, err) diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 9ea621e7..9d139945 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -285,7 +285,8 @@ func (s *Rest) routes() chi.Router { radmin.Get("/export", s.adminRest.migrator.exportCtrl) radmin.Post("/import", s.adminRest.migrator.importCtrl) radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl) - radmin.Get("/import/wait", s.adminRest.migrator.importWaitCtrl) + radmin.Post("/remap", s.adminRest.migrator.remapCtrl) + radmin.Get("/wait", s.adminRest.migrator.waitCtrl) }) // protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 22e83122..4fecd87e 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -330,7 +330,8 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { WordPressImporter: &migrator.WordPress{DataStore: dataStore}, NativeImporter: &migrator.Native{DataStore: dataStore}, NativeExporter: &migrator.Native{DataStore: dataStore}, - Cache: &cache.Nop{}, + UrlMapperMaker: migrator.NewUrlMapper, + Cache: memCache, KeyStore: astore, }, Streamer: &Streamer{ diff --git a/backend/go.mod b/backend/go.mod index 6aae1eed..4b6e33b6 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -56,5 +56,6 @@ require ( golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 golang.org/x/image v0.0.0-20190823064033-3a9bac650e44 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/russross/blackfriday.v2 v2.0.1 )