Dmitry VerkhoturovandGitHub
389189afcf
The test passed one strings.Reader as the body of both POSTs. client.Do returns once the response headers arrive, and the import answers 202 before the transport has finished copying the body, so the second http.NewRequest reads the reader's Len to set ContentLength while the first request's writeLoop is still advancing it. The race detector caught it on CI as a write in strings.(*Reader).WriteTo against a read in NewRequestWithContext, failing a test nothing had touched. Reproduced in isolation to confirm the mechanism rather than infer it from the trace: a handler that answers 202 without draining an 8 MiB body, two requests sharing one reader, and -race reports strings.(*Reader).Len in NewRequestWithContext against strings.(*Reader).Read on every run. It does not reproduce in this package locally, which is why it reads as a flake. Both requests now build their own reader over the same content. The second one carries a full body where before it inherited a consumed one, which is closer to what the case is about: a second import arriving while the first is running still has to be refused.
768 lines
34 KiB
Go
768 lines
34 KiB
Go
package api
|
||
|
||
import (
|
||
"bytes"
|
||
"compress/gzip"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"mime/multipart"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/stretchr/testify/assert"
|
||
"github.com/stretchr/testify/require"
|
||
|
||
"github.com/umputun/remark42/backend/app/store"
|
||
"github.com/umputun/remark42/backend/app/store/service"
|
||
)
|
||
|
||
func TestMigrator_Import(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
|
||
"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
|
||
"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
|
||
"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
|
||
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
|
||
"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
|
||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
|
||
"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
assert.NoError(t, err)
|
||
resp, err := client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
|
||
b, err := io.ReadAll(resp.Body)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||
assert.NoError(t, resp.Body.Close())
|
||
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
}
|
||
|
||
func TestMigrator_ImportForm(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
|
||
"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
|
||
"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
|
||
"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
|
||
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
|
||
"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
|
||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
|
||
"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||
|
||
bodyBuf := &bytes.Buffer{}
|
||
bodyWriter := multipart.NewWriter(bodyBuf)
|
||
fileWriter, err := bodyWriter.CreateFormFile("file", "import.json")
|
||
require.NoError(t, err)
|
||
_, err = io.Copy(fileWriter, r)
|
||
require.NoError(t, err)
|
||
contentType := bodyWriter.FormDataContentType()
|
||
require.NoError(t, bodyWriter.Close())
|
||
|
||
authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
|
||
resp, err := http.Post(authts+"/api/v1/admin/import/form?site=remark42&provider=native", contentType, bodyBuf)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
|
||
b, err := io.ReadAll(resp.Body)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||
assert.NoError(t, resp.Body.Close())
|
||
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
}
|
||
|
||
func TestMigrator_ImportFromWP(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
r := strings.NewReader(strings.ReplaceAll(xmlTestWP, "'", "`"))
|
||
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=wordpress", r)
|
||
assert.NoError(t, err)
|
||
req.Header.Add("Content-Type", "application/xml; charset=utf-8")
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err := client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
|
||
b, err := io.ReadAll(resp.Body)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||
assert.NoError(t, resp.Body.Close())
|
||
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://realmenweardress.es/2010/07/do-you-rp/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 3, comments.Info.Count)
|
||
require.Equal(t, 3, len(comments.Comments))
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://realmenweardress.es/2010/07/do-you-rp/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 3, comments.Info.Count)
|
||
require.Equal(t, 2, len(comments.Comments), "2 comments with 1 reply")
|
||
}
|
||
|
||
func TestMigrator_ImportFromCommento(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
r := strings.NewReader(`{"version":1,"comments":[{"commentHex":"7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
|
||
"domain":"example.com","url":"/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
|
||
"markdown":"Example content","html":"","parentHex":"root","score":0,"state":"approved","creationDate":"2021-03-17T12:09:47.722181Z",
|
||
"direction":0,"deleted":false}],"commenters":[{"commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
|
||
"email":"somegreatmail@gmail.com","name":"User5276","link":"https://example.com/profile/257","photo":"https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
|
||
"provider":"sso:example.com","joinDate":"2021-03-19T19:27:25.954285Z","isModerator":false}]}`)
|
||
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
|
||
assert.NoError(t, err)
|
||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err := client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
|
||
b, err := io.ReadAll(resp.Body)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||
assert.NoError(t, resp.Body.Close())
|
||
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/blog/post/1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/blog/post/1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
}
|
||
|
||
func TestMigrator_ImportFromCommentoJSON(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
r, err := os.Open("testdata/commento.json")
|
||
require.NoError(t, err)
|
||
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
|
||
assert.NoError(t, err)
|
||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err := client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
|
||
b, err := io.ReadAll(resp.Body)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
|
||
assert.NoError(t, resp.Body.Close())
|
||
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 7, comments.Info.Count)
|
||
require.Equal(t, 7, len(comments.Comments))
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/example")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 7, comments.Info.Count)
|
||
require.Equal(t, 5, len(comments.Comments), "five comments with two replies")
|
||
}
|
||
|
||
func TestMigrator_ImportRejected(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
|
||
"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
|
||
"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
|
||
"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
|
||
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
|
||
"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
|
||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
|
||
"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native&secret=XYZ", r)
|
||
assert.NoError(t, err)
|
||
resp, err := client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.NoError(t, resp.Body.Close())
|
||
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||
}
|
||
|
||
func TestMigrator_ImportDouble(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
|
||
"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
|
||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
|
||
"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
|
||
recs := make([]string, 0, 50)
|
||
for i := range 50 {
|
||
recs = append(recs, fmt.Sprintf(tmpl, i))
|
||
}
|
||
// each request needs its own reader. client.Do returns once the response headers are in, which
|
||
// for an accepted import is before the transport's writeLoop has finished copying the body, so
|
||
// handing the same strings.Reader to the second NewRequest races that copy: NewRequest reads
|
||
// Len() to set ContentLength while WriteTo is still advancing it
|
||
body := `{"version":1}` + strings.Join(recs, "\n")
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", strings.NewReader(body))
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
assert.NoError(t, err)
|
||
resp, err := client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.NoError(t, resp.Body.Close())
|
||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
|
||
client = &http.Client{Timeout: 5 * time.Second}
|
||
defer client.CloseIdleConnections()
|
||
req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", strings.NewReader(body))
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
assert.NoError(t, err)
|
||
resp, err = client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.NoError(t, resp.Body.Close())
|
||
assert.Equal(t, http.StatusConflict, resp.StatusCode)
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 50, comments.Info.Count)
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 50, comments.Info.Count)
|
||
}
|
||
|
||
func TestMigrator_ImportWaitExpired(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
|
||
"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
|
||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
|
||
"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
|
||
nRecs := 50
|
||
recs := make([]string, 0, nRecs)
|
||
for i := range nRecs {
|
||
recs = append(recs, fmt.Sprintf(tmpl, i))
|
||
}
|
||
r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with `nRecs` records
|
||
client := &http.Client{Timeout: 5 * time.Second}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
require.NoError(t, err)
|
||
resp, err := client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.NoError(t, resp.Body.Close())
|
||
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
|
||
client = &http.Client{Timeout: 5 * time.Second}
|
||
defer client.CloseIdleConnections()
|
||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=5ms", http.NoBody)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
assert.NoError(t, err)
|
||
resp, err = client.Do(req)
|
||
assert.NoError(t, err)
|
||
assert.NoError(t, resp.Body.Close())
|
||
assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
|
||
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 0, comments.Info.Count)
|
||
require.Equal(t, 0, len(comments.Comments))
|
||
}
|
||
|
||
func TestMigrator_Export(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
|
||
"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
|
||
"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
|
||
"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
|
||
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
|
||
"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
|
||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
|
||
"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||
|
||
// import comments first
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err := client.Do(req)
|
||
require.NoError(t, err)
|
||
require.NoError(t, resp.Body.Close())
|
||
require.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
// export unknown site is a client error, not internal
|
||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=test", http.NoBody)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err = client.Do(req)
|
||
require.NoError(t, err)
|
||
errBody, err := io.ReadAll(resp.Body)
|
||
require.NoError(t, err)
|
||
resp.Body.Close()
|
||
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
|
||
require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
|
||
assert.Contains(t, string(errBody), `"code":6`) // rest.ErrSiteNotFound, not ErrInternal
|
||
assert.Contains(t, string(errBody), `not found`) // error detail names the missing site
|
||
|
||
// unknown site in stream mode is also a client error
|
||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=test", http.NoBody)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err = client.Do(req)
|
||
require.NoError(t, err)
|
||
errBody, err = io.ReadAll(resp.Body)
|
||
require.NoError(t, err)
|
||
resp.Body.Close()
|
||
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
|
||
assert.Contains(t, string(errBody), `"code":6`)
|
||
|
||
// check file mode
|
||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", http.NoBody)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err = client.Do(req)
|
||
require.NoError(t, err)
|
||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||
require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
|
||
|
||
ungzReader, err := gzip.NewReader(resp.Body)
|
||
assert.NoError(t, err)
|
||
ungzBody, err := io.ReadAll(ungzReader)
|
||
assert.NoError(t, err)
|
||
require.NoError(t, resp.Body.Close())
|
||
assert.Equal(t, 3, strings.Count(string(ungzBody), "\n"))
|
||
assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
|
||
t.Logf("%s", string(ungzBody))
|
||
|
||
// check stream mode
|
||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=remark42", http.NoBody)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err = client.Do(req)
|
||
require.NoError(t, err)
|
||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||
require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
|
||
|
||
body, err := io.ReadAll(resp.Body)
|
||
assert.NoError(t, err)
|
||
require.NoError(t, resp.Body.Close())
|
||
assert.Equal(t, 3, strings.Count(string(body), "\n"))
|
||
assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
|
||
t.Logf("%s", string(body))
|
||
|
||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=remark42", http.NoBody)
|
||
require.NoError(t, err)
|
||
resp, err = client.Do(req)
|
||
require.NoError(t, err)
|
||
require.NoError(t, resp.Body.Close())
|
||
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||
}
|
||
|
||
func TestMigrator_Remap(t *testing.T) {
|
||
ts, srv, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
// create 2 comments in https://remark42.com/demo/
|
||
c1 := store.Comment{Text: "first comment", Timestamp: time.Now(),
|
||
Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u1"}}
|
||
_, err := srv.DataService.Create(c1)
|
||
require.NoError(t, err)
|
||
c2 := store.Comment{Text: "second comment", Timestamp: time.Now(),
|
||
Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u2"}}
|
||
_, err = srv.DataService.Create(c2)
|
||
require.NoError(t, err)
|
||
|
||
// create 1 comment in https://remark42.com/demo-another/
|
||
c3 := store.Comment{Text: "third comment", Timestamp: time.Now(),
|
||
Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo-another/"}, User: store.User{ID: "u3"}}
|
||
_, err = srv.DataService.Create(c3)
|
||
require.NoError(t, err)
|
||
|
||
// set url https://remark42.com/demo-another/ to be readonly
|
||
err = srv.DataService.SetMetas("remark42", []service.UserMetaData{}, []service.PostMetaData{{
|
||
URL: "https://remark42.com/demo-another/",
|
||
ReadOnly: true,
|
||
}})
|
||
require.NoError(t, err)
|
||
|
||
// check that comments created as expected
|
||
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments := commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 2, comments.Info.Count)
|
||
require.Equal(t, 2, len(comments.Comments))
|
||
require.False(t, comments.Info.ReadOnly)
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
require.True(t, comments.Info.ReadOnly)
|
||
|
||
// we want remap urls to another domain - www.remark42.com
|
||
rules := "https://remark42.com/* https://www.remark42.com/*"
|
||
resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
|
||
require.NoError(t, err)
|
||
require.NoError(t, resp.Body.Close())
|
||
require.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||
waitForMigrationCompletion(t, ts)
|
||
|
||
// after remap finished we should find comments from new urls
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 2, comments.Info.Count)
|
||
require.Equal(t, 2, len(comments.Comments))
|
||
require.False(t, comments.Info.ReadOnly)
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 2, comments.Info.Count)
|
||
require.Equal(t, 2, len(comments.Comments))
|
||
require.False(t, comments.Info.ReadOnly)
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
require.True(t, comments.Info.ReadOnly)
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo-another/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, comments.Info.Count)
|
||
require.Equal(t, 1, len(comments.Comments))
|
||
require.True(t, comments.Info.ReadOnly)
|
||
|
||
// should find nothing from previous url
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 0, comments.Info.Count)
|
||
require.Equal(t, 0, len(comments.Comments))
|
||
|
||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
|
||
require.Equal(t, http.StatusOK, code)
|
||
comments = commentsWithInfo{}
|
||
err = json.Unmarshal([]byte(res), &comments)
|
||
require.NoError(t, err)
|
||
require.Equal(t, 0, comments.Info.Count)
|
||
require.Equal(t, 0, len(comments.Comments))
|
||
}
|
||
|
||
func TestMigrator_RemapReject(t *testing.T) {
|
||
ts, _, teardown := startupT(t)
|
||
defer teardown()
|
||
|
||
// without admin credentials
|
||
client := &http.Client{Timeout: waitTimeout}
|
||
defer client.CloseIdleConnections()
|
||
rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`)
|
||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules)
|
||
require.NoError(t, err)
|
||
resp, err := client.Do(req)
|
||
require.NoError(t, err)
|
||
require.NoError(t, resp.Body.Close())
|
||
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||
}
|
||
|
||
func waitForMigrationCompletion(t *testing.T, ts *httptest.Server) {
|
||
client := &http.Client{Timeout: 10 * time.Second}
|
||
defer client.CloseIdleConnections()
|
||
req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42", http.NoBody)
|
||
require.NoError(t, err)
|
||
req.SetBasicAuth("admin", "password")
|
||
resp, err := client.Do(req)
|
||
require.NoError(t, err)
|
||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||
b, err := io.ReadAll(resp.Body)
|
||
require.NoError(t, err)
|
||
defer resp.Body.Close()
|
||
assert.Equal(t, "{\"site_id\":\"remark42\",\"status\":\"completed\"}\n", string(b))
|
||
}
|
||
|
||
var xmlTestWP = `
|
||
<?xml version="1.0" encoding="UTF-8" ?>
|
||
<rss version="2.0"
|
||
xmlns:excerpt="http://wordpress.org/export/1.2/excerpt/"
|
||
xmlns:content="http://purl.org/rss/1.0/modules/content/"
|
||
xmlns:wfw="http://wellformedweb.org/CommentAPI/"
|
||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||
xmlns:wp="http://wordpress.org/export/1.2/"
|
||
>
|
||
|
||
<channel>
|
||
<title>Real Men Wear Dress.es</title>
|
||
<link>https://realmenweardress.es</link>
|
||
<description>SuperAdmin's gaming and technological musings</description>
|
||
<pubDate>Mon, 23 Jul 2018 10:21:47 +0000</pubDate>
|
||
<language>en-US</language>
|
||
<wp:wxr_version>1.2</wp:wxr_version>
|
||
<wp:base_site_url>https://realmenweardress.es</wp:base_site_url>
|
||
<wp:base_blog_url>https://realmenweardress.es</wp:base_blog_url>
|
||
|
||
<wp:author><wp:author_id>2</wp:author_id><wp:author_login><![CDATA[SuperAdmin]]></wp:author_login><wp:author_email><![CDATA[superadmin@super.eu]]></wp:author_email><wp:author_display_name><![CDATA[SuperAdmin]]></wp:author_display_name><wp:author_first_name><![CDATA[SuperAdmin]]></wp:author_first_name><wp:author_last_name><![CDATA[superadmin]]></wp:author_last_name></wp:author>
|
||
<wp:author><wp:author_id>1</wp:author_id><wp:author_login><![CDATA[admin]]></wp:author_login><wp:author_email><![CDATA[superadmin@superadmin.co.uk]]></wp:author_email><wp:author_display_name><![CDATA[admin]]></wp:author_display_name><wp:author_first_name><![CDATA[]]></wp:author_first_name><wp:author_last_name><![CDATA[]]></wp:author_last_name></wp:author>
|
||
|
||
<wp:category>
|
||
<wp:term_id>25</wp:term_id>
|
||
<wp:category_nicename><![CDATA[cataclysm]]></wp:category_nicename>
|
||
<wp:category_parent><![CDATA[]]></wp:category_parent>
|
||
<wp:cat_name><![CDATA[Cataclysm]]></wp:cat_name>
|
||
</wp:category>
|
||
|
||
<wp:tag>
|
||
<wp:term_id>39</wp:term_id>
|
||
<wp:tag_slug><![CDATA[addons]]></wp:tag_slug>
|
||
<wp:tag_name><![CDATA[addons]]></wp:tag_name>
|
||
</wp:tag>
|
||
|
||
<generator>https://wordpress.org/?v=4.8.1</generator>
|
||
|
||
<item>
|
||
<title>Post without comments</title>
|
||
<link>https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/</link>
|
||
<pubDate>Sat, 19 Jun 2010 08:34:13 +0000</pubDate>
|
||
<dc:creator><![CDATA[admin]]></dc:creator>
|
||
<guid isPermaLink="false">http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg</guid>
|
||
<description></description>
|
||
<content:encoded><![CDATA[So you can actually fly into the well it appears and if your lucky you stay mounted. I imagine it terrifies the poor rats.]]></content:encoded>
|
||
<excerpt:encoded><![CDATA[]]></excerpt:encoded>
|
||
<wp:post_id>6</wp:post_id>
|
||
<wp:post_date><![CDATA[2010-06-19 08:34:13]]></wp:post_date>
|
||
<wp:post_date_gmt><![CDATA[2010-06-19 08:34:13]]></wp:post_date_gmt>
|
||
<wp:comment_status><![CDATA[open]]></wp:comment_status>
|
||
<wp:ping_status><![CDATA[open]]></wp:ping_status>
|
||
<wp:post_name><![CDATA[screenshot_013110_200413]]></wp:post_name>
|
||
<wp:status><![CDATA[inherit]]></wp:status>
|
||
<wp:post_parent>1</wp:post_parent>
|
||
<wp:menu_order>0</wp:menu_order>
|
||
<wp:post_type><![CDATA[attachment]]></wp:post_type>
|
||
<wp:post_password><![CDATA[]]></wp:post_password>
|
||
<wp:is_sticky>0</wp:is_sticky>
|
||
<wp:attachment_url><![CDATA[https://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:attachment_url>
|
||
<wp:postmeta>
|
||
<wp:meta_key><![CDATA[_wp_attached_file]]></wp:meta_key>
|
||
<wp:meta_value><![CDATA[2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:meta_value>
|
||
</wp:postmeta>
|
||
</item>
|
||
<item>
|
||
<title>Post with comments. One is not approved</title>
|
||
<link>https://realmenweardress.es/2010/07/do-you-rp/</link>
|
||
<pubDate>Mon, 19 Jul 2010 14:24:22 +0000</pubDate>
|
||
<dc:creator><![CDATA[SuperAdmin]]></dc:creator>
|
||
<guid isPermaLink="false">http://realmenweardress.es/?p=100</guid>
|
||
<description></description>
|
||
<content:encoded><![CDATA[<a href="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546180886.jpeg"><img class="size-thumbnail wp-image-102 alignleft" title="I need to stand on things else I can't reach" src="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546270587-120x120.jpg" alt="I need to stand on things else I can't reach" width="120" height="120" /></a>Meet Grokknomel?]]></content:encoded>
|
||
<excerpt:encoded><![CDATA[]]></excerpt:encoded>
|
||
<wp:post_id>100</wp:post_id>
|
||
<wp:post_date><![CDATA[2010-07-19 14:24:22]]></wp:post_date>
|
||
<wp:post_date_gmt><![CDATA[2010-07-19 14:24:22]]></wp:post_date_gmt>
|
||
<wp:comment_status><![CDATA[open]]></wp:comment_status>
|
||
<wp:ping_status><![CDATA[open]]></wp:ping_status>
|
||
<wp:post_name><![CDATA[do-you-rp]]></wp:post_name>
|
||
<wp:status><![CDATA[publish]]></wp:status>
|
||
<wp:post_parent>0</wp:post_parent>
|
||
<wp:menu_order>0</wp:menu_order>
|
||
<wp:post_type><![CDATA[post]]></wp:post_type>
|
||
<wp:post_password><![CDATA[]]></wp:post_password>
|
||
<wp:is_sticky>0</wp:is_sticky>
|
||
<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
|
||
<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
|
||
<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
|
||
<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
|
||
<wp:postmeta>
|
||
<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
|
||
<wp:meta_value><![CDATA[2]]></wp:meta_value>
|
||
</wp:postmeta>
|
||
<wp:comment>
|
||
<wp:comment_id>8</wp:comment_id>
|
||
<wp:comment_author><![CDATA[SuperUser1]]></wp:comment_author>
|
||
<wp:comment_author_email><![CDATA[superuser1@aol.com]]></wp:comment_author_email>
|
||
<wp:comment_author_url>http://superuser1.blogspot.com</wp:comment_author_url>
|
||
<wp:comment_author_IP><![CDATA[79.141.141.73]]></wp:comment_author_IP>
|
||
<wp:comment_date><![CDATA[2010-07-20 12:08:08]]></wp:comment_date>
|
||
<wp:comment_date_gmt><![CDATA[2010-07-20 12:08:08]]></wp:comment_date_gmt>
|
||
<wp:comment_content><![CDATA[I do catch myself]]></wp:comment_content>
|
||
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
|
||
<wp:comment_type><![CDATA[]]></wp:comment_type>
|
||
<wp:comment_parent>0</wp:comment_parent>
|
||
<wp:comment_user_id>0</wp:comment_user_id>
|
||
</wp:comment>
|
||
<wp:comment>
|
||
<wp:comment_id>9</wp:comment_id>
|
||
<wp:comment_author><![CDATA[SuperUser2]]></wp:comment_author>
|
||
<wp:comment_author_email><![CDATA[superuser2@gmail.com]]></wp:comment_author_email>
|
||
<wp:comment_author_url>http://thewowstorm.wordpress.com</wp:comment_author_url>
|
||
<wp:comment_author_IP><![CDATA[97.36.113.1]]></wp:comment_author_IP>
|
||
<wp:comment_date><![CDATA[2010-07-20 13:09:25]]></wp:comment_date>
|
||
<wp:comment_date_gmt><![CDATA[2010-07-20 13:09:25]]></wp:comment_date_gmt>
|
||
<wp:comment_content><![CDATA[I think it us inherent in the game to start seeing your character as a personality]]></wp:comment_content>
|
||
<wp:comment_approved><![CDATA[0]]></wp:comment_approved>
|
||
<wp:comment_type><![CDATA[]]></wp:comment_type>
|
||
<wp:comment_parent>0</wp:comment_parent>
|
||
<wp:comment_user_id>0</wp:comment_user_id>
|
||
</wp:comment>
|
||
<wp:comment>
|
||
<wp:comment_id>13</wp:comment_id>
|
||
<wp:comment_author><![CDATA[Wednesday Reading « Cynwise's Battlefield Manual]]></wp:comment_author>
|
||
<wp:comment_author_email><![CDATA[]]></wp:comment_author_email>
|
||
<wp:comment_author_url>http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/</wp:comment_author_url>
|
||
<wp:comment_author_IP><![CDATA[74.200.244.101]]></wp:comment_author_IP>
|
||
<wp:comment_date><![CDATA[2010-07-21 14:02:08]]></wp:comment_date>
|
||
<wp:comment_date_gmt><![CDATA[2010-07-21 14:02:08]]></wp:comment_date_gmt>
|
||
<wp:comment_content><![CDATA[[...] I know I’m a bit loony with my attachment to my bankers. I’m glad I’m not the only one. [...]]]></wp:comment_content>
|
||
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
|
||
<wp:comment_type><![CDATA[pingback]]></wp:comment_type>
|
||
<wp:comment_parent>0</wp:comment_parent>
|
||
<wp:comment_user_id>0</wp:comment_user_id>
|
||
</wp:comment>
|
||
<wp:comment>
|
||
<wp:comment_id>14</wp:comment_id>
|
||
<wp:comment_author><![CDATA[SuperUser3]]></wp:comment_author>
|
||
<wp:comment_author_email><![CDATA[blablah@gmail.com]]></wp:comment_author_email>
|
||
<wp:comment_author_url>http://realmenweardress.es</wp:comment_author_url>
|
||
<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
|
||
<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
|
||
<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
|
||
<wp:comment_content><![CDATA[Looks like http://releases.rancher.com/os/latest is no longer hosted - installs using this 'base-url' are failing.
|
||
|
||
I switched to Github with success:
|
||
|
||
'''
|
||
set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1
|
||
'''
|
||
|
||
Thanks for the article!]]></wp:comment_content>
|
||
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
|
||
<wp:comment_type><![CDATA[]]></wp:comment_type>
|
||
<wp:comment_parent>13</wp:comment_parent>
|
||
<wp:comment_user_id>2</wp:comment_user_id>
|
||
</wp:comment>
|
||
</item>
|
||
</channel>
|
||
</rss>
|
||
`
|