Change url in comments, decorate approach (#431)
* WIP: url mapper, wrapped reader approach * create url mapper on start * add pattern matching in mapper * check pattern matching in test * change site-id from radio-t to remark42 in tests * create new url mapper on demand, based on given func, union strict and prefix rules * rename convert to remap * add import with mapper test * rename mapper func to UrlMapperMaker, create comments in test via data service * move /import/wait to /wait ctrl * add remap cmd * fix url naming
This commit is contained in:
Anton Kosourov
Umputun
@@ -769,7 +769,14 @@ _returned id should be appended to load image url on caller side_
|
||||
* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
|
||||
* `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
|
||||
* `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
|
||||
* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completion.
|
||||
* `POST /api/v1/admin/remap?site=side-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n.
|
||||
From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on
|
||||
export/import chain so make backup first.
|
||||
```
|
||||
http://oldsite.com* https://newsite.com*
|
||||
http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
|
||||
```
|
||||
* `GET /api/v1/admin/wait?site=side-id` - wait for completion for any async migration ops (import or remap).
|
||||
* `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
|
||||
* `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
|
||||
* `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// RemapCommand set of flags and command for change linkage between comments to
|
||||
// different urls based on given rules (input file)
|
||||
type RemapCommand struct {
|
||||
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
|
||||
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
|
||||
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
|
||||
Timeout time.Duration `long:"timeout" default:"15m" description:"remap timeout"`
|
||||
CommonOpts
|
||||
}
|
||||
|
||||
func (rc *RemapCommand) Execute(args []string) error {
|
||||
log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
|
||||
resetEnv("SECRET", "ADMIN_PASSWD")
|
||||
|
||||
rulesReader, err := os.Open(rc.InputFile)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "cant open file %s", rc.InputFile)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout)
|
||||
defer cancel()
|
||||
remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site)
|
||||
req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "can't make remap request for %s", remapURL)
|
||||
}
|
||||
req.SetBasicAuth("admin", rc.AdminPasswd)
|
||||
|
||||
resp, err := client.Do(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "request failed for %s", remapURL)
|
||||
}
|
||||
defer func() {
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
log.Printf("[WARN] failed to close response, %s", err)
|
||||
}
|
||||
}()
|
||||
if resp.StatusCode >= 300 {
|
||||
return responseError(resp)
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "can't get response")
|
||||
}
|
||||
|
||||
log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRemap_Execute(t *testing.T) {
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
assert.Equal(t, "remark", r.URL.Query().Get("site"))
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
|
||||
|
||||
w.WriteHeader(202)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
cmd := RemapCommand{}
|
||||
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
|
||||
|
||||
p := flags.NewParser(&cmd, flags.Default)
|
||||
_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt", "--admin-passwd=secret"})
|
||||
require.Nil(t, err)
|
||||
err = cmd.Execute(nil)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
@@ -305,6 +305,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
|
||||
DisqusImporter: &migrator.Disqus{DataStore: dataService},
|
||||
WordPressImporter: &migrator.WordPress{DataStore: dataService},
|
||||
NativeExporter: &migrator.Native{DataStore: dataService},
|
||||
UrlMapperMaker: migrator.NewUrlMapper,
|
||||
KeyStore: adminStore,
|
||||
}
|
||||
|
||||
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
http://oldsite.com* https://newsite.com*
|
||||
http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
|
||||
@@ -21,6 +21,7 @@ type Opts struct {
|
||||
RestoreCmd cmd.RestoreCommand `command:"restore"`
|
||||
AvatarCmd cmd.AvatarCommand `command:"avatar"`
|
||||
CleanupCmd cmd.CleanupCommand `command:"cleanup"`
|
||||
RemapCmd cmd.RemapCommand `command:"remap"`
|
||||
|
||||
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
|
||||
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// UrlMapper implements Mapper interface
|
||||
type UrlMapper struct {
|
||||
rules map[string]string
|
||||
}
|
||||
|
||||
// NewUrlMapper reads rules from given reader and returns initialised UrlMapper
|
||||
// if given rules are valid.
|
||||
func NewUrlMapper(reader io.Reader) (Mapper, error) {
|
||||
u := &UrlMapper{}
|
||||
if err := u.loadRules(reader); err != nil {
|
||||
return u, err
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// loadRules loads url-mapping rules from reader to mapper.
|
||||
// Rules must be a text consists of rows separated by \n.
|
||||
// Each row holds from-url and to-url separated by space.
|
||||
// If urls end with asterisk (*) it means try to match by prefix.
|
||||
// Example:
|
||||
// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
|
||||
// https://www.myblog.com/* https://myblog.com/*
|
||||
func (u *UrlMapper) loadRules(reader io.Reader) error {
|
||||
data, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rulesText := strings.TrimSpace(string(data))
|
||||
|
||||
u.rules = make(map[string]string)
|
||||
|
||||
for _, row := range strings.Split(rulesText, "\n") {
|
||||
row = strings.TrimSpace(row)
|
||||
urls := strings.Split(row, " ")
|
||||
if len(urls) != 2 {
|
||||
return errors.New("bad row " + row)
|
||||
}
|
||||
|
||||
from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1])
|
||||
u.rules[from] = to
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// URL maps given url to another url according loaded url-rules.
|
||||
// If not matched returns given url.
|
||||
func (u *UrlMapper) URL(url string) string {
|
||||
if newUrl, ok := u.rules[url]; ok {
|
||||
return newUrl
|
||||
}
|
||||
// try to match by prefix
|
||||
for oldUrl, newUrl := range u.rules {
|
||||
if !strings.HasSuffix(oldUrl, "*") {
|
||||
continue
|
||||
}
|
||||
oldUrl = strings.TrimSuffix(oldUrl, "*")
|
||||
newUrl = strings.TrimSuffix(newUrl, "*")
|
||||
if strings.HasPrefix(url, oldUrl) {
|
||||
return newUrl + strings.TrimPrefix(url, oldUrl)
|
||||
}
|
||||
}
|
||||
// search failed, return given url
|
||||
return url
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestUrlMapper_URL(t *testing.T) {
|
||||
// want remap urls from https://radio-t.com to https://www.radio-t.com
|
||||
// also map individual urls
|
||||
rules := strings.NewReader(`
|
||||
https://radio-t.com* https://www.radio-t.com*
|
||||
https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/09/22/podcast-616/
|
||||
https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
|
||||
`)
|
||||
|
||||
mapper, err := NewUrlMapper(rules)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// if url not matched mapper should return given url
|
||||
assert.Equal(t, "https://any.com/post/1/", mapper.URL("https://any.com/post/1/"))
|
||||
assert.Equal(t, "https://radio-t.co", mapper.URL("https://radio-t.co"))
|
||||
// check strict matching
|
||||
assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22////podcast-616/"))
|
||||
assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1"))
|
||||
// check pattern matching (by prefix)
|
||||
assert.Equal(t, "https://www.radio-t.com/p/post/123/", mapper.URL("https://radio-t.com/p/post/123/"))
|
||||
|
||||
// want remap from http to https
|
||||
rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
|
||||
mapper, err = NewUrlMapper(rules)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
|
||||
assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
|
||||
assert.Equal(t, "https://anysite.com/", mapper.URL("https://anysite.com/")) // not exist
|
||||
|
||||
// want remap from http to https by pattern
|
||||
rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
|
||||
mapper, err = NewUrlMapper(rules)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
|
||||
assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
|
||||
assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
|
||||
}
|
||||
|
||||
func TestUrlMapper_New(t *testing.T) {
|
||||
cases := []struct {
|
||||
rules string
|
||||
expectError bool
|
||||
}{
|
||||
// bad input, expect error
|
||||
{
|
||||
rules: "https://radio-t.com ",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
rules: "https://radio-t.com https://radio-t.com https://radio-t.com",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
rules: "https://radio-t.com https://radio-t.com\n https://radio-t.com",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
rules: "https://radio-t.com \n https://radio-t.com https://radio-t.com",
|
||||
expectError: true,
|
||||
},
|
||||
|
||||
// valid input, no error
|
||||
{
|
||||
rules: "https://radio-t.com* https://www.radio-t.com*",
|
||||
},
|
||||
{
|
||||
rules: "https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/",
|
||||
},
|
||||
{
|
||||
rules: "https://any.com/p/111 https://any.com/p/222 \n https://any.com/p/333 https://any.com/p/222 \n",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
_, err := NewUrlMapper(strings.NewReader(c.rules))
|
||||
if c.expectError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,16 @@ type Exporter interface {
|
||||
Export(w io.Writer, siteID string) (int, error)
|
||||
}
|
||||
|
||||
// Mapper defines interface to convert data in import procedure
|
||||
type Mapper interface {
|
||||
URL(url string) string
|
||||
}
|
||||
|
||||
// MapperMaker defines function that reads rules from reader and
|
||||
// returns new Mapper with loaded rules. If rules are not valid
|
||||
// it returns error.
|
||||
type MapperMaker func(reader io.Reader) (Mapper, error)
|
||||
|
||||
// Store defines minimal interface needed to export and import comments
|
||||
type Store interface {
|
||||
Create(comment store.Comment) (commentID string, err error)
|
||||
|
||||
@@ -87,9 +87,49 @@ func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// WithMapper wraps reader with url-mapper.
|
||||
func WithMapper(reader io.Reader, mapper Mapper) io.Reader {
|
||||
r, w := io.Pipe()
|
||||
go func() {
|
||||
var err error
|
||||
defer func() {
|
||||
log.Printf("[DEBUG] finish write to pipe with %+v", err)
|
||||
if e := w.Close(); e != nil {
|
||||
log.Printf("[WARN] failed close pipe writer with %+v", e)
|
||||
}
|
||||
}()
|
||||
|
||||
// decode from reader and encode to pipe writer
|
||||
dec, enc := json.NewDecoder(reader), json.NewEncoder(w)
|
||||
|
||||
m := meta{}
|
||||
if err = dec.Decode(&m); err != nil {
|
||||
return
|
||||
}
|
||||
for i := range m.Posts {
|
||||
m.Posts[i].URL = mapper.URL(m.Posts[i].URL)
|
||||
}
|
||||
if err = enc.Encode(m); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
comment := store.Comment{}
|
||||
if err = dec.Decode(&comment); err != nil {
|
||||
return
|
||||
}
|
||||
comment.Locator.URL = mapper.URL(comment.Locator.URL)
|
||||
if err = enc.Encode(comment); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// Import comments from json strings produced by Remark.Export
|
||||
func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
|
||||
|
||||
m := meta{}
|
||||
dec := json.NewDecoder(reader)
|
||||
if err = dec.Decode(&m); err != nil {
|
||||
|
||||
@@ -101,6 +101,45 @@ func TestNative_Import(t *testing.T) {
|
||||
assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
|
||||
}
|
||||
|
||||
func TestNative_ImportWithMapper(t *testing.T) {
|
||||
defer os.Remove(testDb)
|
||||
|
||||
// want to remap comments to https://rdt.c
|
||||
rules := `https://radio-t.com* https://rdt.c*`
|
||||
mapper, err := NewUrlMapper(strings.NewReader(rules))
|
||||
assert.NoError(t, err)
|
||||
|
||||
inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
|
||||
{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
|
||||
{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
|
||||
mappedReader := WithMapper(strings.NewReader(inp), mapper)
|
||||
|
||||
b := prep(t) // write some recs, they will be deleted
|
||||
b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
|
||||
r := Native{DataStore: b}
|
||||
size, err := r.Import(mappedReader, "radio-t")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, size)
|
||||
|
||||
comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(comments))
|
||||
assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
|
||||
assert.Equal(t, "1234", comments[0].ParentID)
|
||||
assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
|
||||
assert.Equal(t, "https://rdt.c/2", comments[0].Locator.URL)
|
||||
|
||||
assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
|
||||
assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
|
||||
assert.Equal(t, "https://rdt.c", comments[1].Locator.URL)
|
||||
|
||||
assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
|
||||
assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
|
||||
|
||||
assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
|
||||
assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
|
||||
}
|
||||
|
||||
func TestNative_ImportWrongVersion(t *testing.T) {
|
||||
inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
|
||||
{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
|
||||
|
||||
@@ -28,6 +28,7 @@ type Migrator struct {
|
||||
DisqusImporter migrator.Importer
|
||||
WordPressImporter migrator.Importer
|
||||
NativeExporter migrator.Exporter
|
||||
UrlMapperMaker migrator.MapperMaker
|
||||
KeyStore KeyStore
|
||||
|
||||
busy map[string]bool
|
||||
@@ -98,7 +99,9 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
render.JSON(w, r, R.JSON{"status": "import request accepted"})
|
||||
}
|
||||
|
||||
func (m *Migrator) importWaitCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
// GET /wait?site=site-id
|
||||
// waits for migration operation (import or remap)
|
||||
func (m *Migrator) waitCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
siteID := r.URL.Query().Get("site")
|
||||
timeOut := time.Minute * 15
|
||||
if v := r.URL.Query().Get("timeout"); v != "" {
|
||||
@@ -152,6 +155,62 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// POST /remap?site=site-id
|
||||
// remap urls in comments based on given rules (oldUrl newUrl)
|
||||
func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
siteID := r.URL.Query().Get("site")
|
||||
|
||||
// create new url-mapper from given rules in body
|
||||
mapper, err := m.UrlMapperMaker(r.Body)
|
||||
if err != nil {
|
||||
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
|
||||
return
|
||||
}
|
||||
defer r.Body.Close()
|
||||
|
||||
// start remap procedure with mapper
|
||||
go func() {
|
||||
m.setBusy(siteID, true)
|
||||
defer m.setBusy(siteID, false)
|
||||
|
||||
// do export
|
||||
fh, err := ioutil.TempFile("", "remark42_convert")
|
||||
if err != nil {
|
||||
log.Printf("[WARN] failed to make temp file %+v", err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := os.Remove(fh.Name()); err != nil {
|
||||
log.Printf("[WARN] failed to remove temp file %+v", err)
|
||||
}
|
||||
}()
|
||||
log.Printf("[DEBUG] start export for site=%s", siteID)
|
||||
if _, err := m.NativeExporter.Export(fh, siteID); err != nil {
|
||||
log.Printf("[WARN] export failed with %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = fh.Seek(0, 0); err != nil {
|
||||
log.Printf("[WARN] failed to seek file %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] start import for site=%s", siteID)
|
||||
mappedReader := migrator.WithMapper(fh, mapper)
|
||||
size, err := m.NativeImporter.Import(mappedReader, siteID)
|
||||
if err != nil {
|
||||
log.Printf("[WARN] import failed with %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
|
||||
log.Printf("[DEBUG] convert request completed. site=%s, comments=%d", siteID, size)
|
||||
}()
|
||||
|
||||
render.Status(r, http.StatusAccepted)
|
||||
render.JSON(w, r, R.JSON{"status": "convert request accepted"})
|
||||
}
|
||||
|
||||
// runImport reads from tmpfile and import for given siteID and provider
|
||||
func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
|
||||
m.setBusy(siteID, true)
|
||||
|
||||
@@ -3,6 +3,7 @@ package api
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -15,6 +16,9 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/service"
|
||||
)
|
||||
|
||||
func TestMigrator_Import(t *testing.T) {
|
||||
@@ -43,7 +47,7 @@ func TestMigrator_Import(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||||
|
||||
waitForImportCompletion(t, ts)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
}
|
||||
|
||||
func TestMigrator_ImportForm(t *testing.T) {
|
||||
@@ -77,7 +81,7 @@ func TestMigrator_ImportForm(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||||
|
||||
waitForImportCompletion(t, ts)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
}
|
||||
|
||||
func TestMigrator_ImportFromWP(t *testing.T) {
|
||||
@@ -99,7 +103,7 @@ func TestMigrator_ImportFromWP(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||||
|
||||
waitForImportCompletion(t, ts)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
}
|
||||
|
||||
func TestMigrator_ImportRejected(t *testing.T) {
|
||||
@@ -153,7 +157,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
|
||||
resp, err = client.Do(req)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, http.StatusConflict, resp.StatusCode)
|
||||
waitForImportCompletion(t, ts)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
}
|
||||
|
||||
func TestMigrator_ImportWaitExpired(t *testing.T) {
|
||||
@@ -179,7 +183,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
|
||||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||||
|
||||
client = &http.Client{Timeout: 10 * time.Second}
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=remark42&timeout=100ms", nil)
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=100ms", nil)
|
||||
require.NoError(t, err)
|
||||
req.SetBasicAuth("admin", "password")
|
||||
assert.NoError(t, err)
|
||||
@@ -187,7 +191,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
|
||||
|
||||
waitForImportCompletion(t, ts)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
}
|
||||
|
||||
func TestMigrator_Export(t *testing.T) {
|
||||
@@ -211,7 +215,7 @@ func TestMigrator_Export(t *testing.T) {
|
||||
resp, err := client.Do(req)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||||
waitForImportCompletion(t, ts)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
|
||||
// check file mode
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", nil)
|
||||
@@ -252,9 +256,107 @@ func TestMigrator_Export(t *testing.T) {
|
||||
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||||
}
|
||||
|
||||
func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
|
||||
func TestMigrator_Remap(t *testing.T) {
|
||||
ts, srv, teardown := startupT(t)
|
||||
defer teardown()
|
||||
|
||||
// create 2 comments in https://remark42.com/demo/
|
||||
c1 := store.Comment{Text: "first comment", Timestamp: time.Now(),
|
||||
Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u1"}}
|
||||
_, err := srv.DataService.Create(c1)
|
||||
require.NoError(t, err)
|
||||
c2 := store.Comment{Text: "second comment", Timestamp: time.Now(),
|
||||
Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u2"}}
|
||||
_, err = srv.DataService.Create(c2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// create 1 comment in https://remark42.com/demo-another/
|
||||
c3 := store.Comment{Text: "third comment", Timestamp: time.Now(),
|
||||
Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo-another/"}, User: store.User{ID: "u3"}}
|
||||
_, err = srv.DataService.Create(c3)
|
||||
require.NoError(t, err)
|
||||
|
||||
// set url https://remark42.com/demo-another/ to be readonly
|
||||
err = srv.DataService.SetMetas("remark42", []service.UserMetaData{}, []service.PostMetaData{{
|
||||
URL: "https://remark42.com/demo-another/",
|
||||
ReadOnly: true,
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
|
||||
// check that comments created as expected
|
||||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
|
||||
require.Equal(t, 200, code)
|
||||
comments := commentsWithInfo{}
|
||||
err = json.Unmarshal([]byte(res), &comments)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 2, comments.Info.Count)
|
||||
require.False(t, comments.Info.ReadOnly)
|
||||
|
||||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
|
||||
require.Equal(t, 200, code)
|
||||
comments = commentsWithInfo{}
|
||||
err = json.Unmarshal([]byte(res), &comments)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 1, comments.Info.Count)
|
||||
require.True(t, comments.Info.ReadOnly)
|
||||
|
||||
// we want remap urls to another domain - www.remark42.com
|
||||
rules := "https://remark42.com/* https://www.remark42.com/*"
|
||||
resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
|
||||
// after remap finished we should find comments from new urls
|
||||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo/")
|
||||
require.Equal(t, 200, code)
|
||||
comments = commentsWithInfo{}
|
||||
err = json.Unmarshal([]byte(res), &comments)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 2, comments.Info.Count)
|
||||
require.False(t, comments.Info.ReadOnly)
|
||||
|
||||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
|
||||
require.Equal(t, 200, code)
|
||||
comments = commentsWithInfo{}
|
||||
err = json.Unmarshal([]byte(res), &comments)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 1, comments.Info.Count)
|
||||
require.True(t, comments.Info.ReadOnly)
|
||||
|
||||
// should find nothing from previous url
|
||||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
|
||||
require.Equal(t, 200, code)
|
||||
comments = commentsWithInfo{}
|
||||
err = json.Unmarshal([]byte(res), &comments)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 0, comments.Info.Count)
|
||||
|
||||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
|
||||
require.Equal(t, 200, code)
|
||||
comments = commentsWithInfo{}
|
||||
err = json.Unmarshal([]byte(res), &comments)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 0, comments.Info.Count)
|
||||
}
|
||||
|
||||
func TestMigrator_RemapReject(t *testing.T) {
|
||||
ts, _, teardown := startupT(t)
|
||||
defer teardown()
|
||||
|
||||
// without admin credentials
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`)
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules)
|
||||
require.Nil(t, err)
|
||||
resp, err := client.Do(req)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||||
}
|
||||
|
||||
func waitForMigrationCompletion(t *testing.T, ts *httptest.Server) {
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=remark42", nil)
|
||||
req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42", nil)
|
||||
require.NoError(t, err)
|
||||
req.SetBasicAuth("admin", "password")
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -285,7 +285,8 @@ func (s *Rest) routes() chi.Router {
|
||||
radmin.Get("/export", s.adminRest.migrator.exportCtrl)
|
||||
radmin.Post("/import", s.adminRest.migrator.importCtrl)
|
||||
radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl)
|
||||
radmin.Get("/import/wait", s.adminRest.migrator.importWaitCtrl)
|
||||
radmin.Post("/remap", s.adminRest.migrator.remapCtrl)
|
||||
radmin.Get("/wait", s.adminRest.migrator.waitCtrl)
|
||||
})
|
||||
|
||||
// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
|
||||
|
||||
@@ -330,7 +330,8 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
|
||||
WordPressImporter: &migrator.WordPress{DataStore: dataStore},
|
||||
NativeImporter: &migrator.Native{DataStore: dataStore},
|
||||
NativeExporter: &migrator.Native{DataStore: dataStore},
|
||||
Cache: &cache.Nop{},
|
||||
UrlMapperMaker: migrator.NewUrlMapper,
|
||||
Cache: memCache,
|
||||
KeyStore: astore,
|
||||
},
|
||||
Streamer: &Streamer{
|
||||
|
||||
@@ -56,5 +56,6 @@ require (
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
|
||||
golang.org/x/image v0.0.0-20190823064033-3a9bac650e44
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/russross/blackfriday.v2 v2.0.1
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user