package api import ( "bytes" "compress/gzip" "encoding/json" "fmt" "io" "io/ioutil" "mime/multipart" "net/http" "net/http/httptest" "os" "strings" "testing" "time" bolt "github.com/coreos/bbolt" "github.com/go-chi/chi" "github.com/go-pkgz/auth" "github.com/go-pkgz/auth/token" "github.com/go-pkgz/rest/cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) func TestMigrator_Import(t *testing.T) { srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"
test test #1
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"test test #2
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r) req.SetBasicAuth("admin", "password") assert.Nil(t, err) resp, err := client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusAccepted, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) client = &http.Client{Timeout: 10 * time.Second} req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) req.SetBasicAuth("admin", "password") assert.NoError(t, err) resp, err = client.Do(req) require.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) } func TestMigrator_ImportForm(t *testing.T) { srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"test test #1
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"test test #2
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) fileWriter, err := bodyWriter.CreateFormFile("file", "import.json") require.NoError(t, err) _, err = io.Copy(fileWriter, r) require.NoError(t, err) contentType := bodyWriter.FormDataContentType() require.NoError(t, bodyWriter.Close()) authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1) resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf) assert.Nil(t, err) assert.Equal(t, http.StatusAccepted, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) client := &http.Client{Timeout: 10 * time.Second} req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) req.SetBasicAuth("admin", "password") assert.NoError(t, err) resp, err = client.Do(req) require.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) } func TestMigrator_ImportFromWP(t *testing.T) { srv, ds, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1)) client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r) assert.Nil(t, err) req.Header.Add("Content-Type", "application/xml; charset=utf-8") req.SetBasicAuth("admin", "password") resp, err := client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusAccepted, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) client = &http.Client{Timeout: 10 * time.Second} req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) req.SetBasicAuth("admin", "password") assert.NoError(t, err) resp, err = client.Do(req) assert.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) assert.NoError(t, ds.Interface.Close()) tsAccess, _, teardownAccess := startupT(t) defer teardownAccess() res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t") require.Equal(t, 200, code) comments := []store.Comment{} err = json.Unmarshal([]byte(res), &comments) assert.Nil(t, err) assert.Equal(t, 3, len(comments), "should have 3 comments") t.Logf("%+v", comments) assert.Equal(t, "Looks like http://releases.rancher.com/os/latest is no longer hosted - installs using this base-url are failing.
I switched to Github with success:
\n\nset base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n\n\nThanks for the article!
\n", comments[0].Text) } func TestMigrator_ImportRejected(t *testing.T) { srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"test test #1
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"test test #2
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r) assert.Nil(t, err) resp, err := client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) } func TestMigrator_ImportDouble(t *testing.T) { srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) tmpl := `{"id":"%d","pid":"","text":"test test #1
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}` recs := []string{} for i := 0; i < 5000; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r) req.SetBasicAuth("admin", "password") assert.Nil(t, err) resp, err := client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusAccepted, resp.StatusCode) client = &http.Client{Timeout: 1 * time.Second} req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r) req.SetBasicAuth("admin", "password") assert.Nil(t, err) resp, err = client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusConflict, resp.StatusCode) } func TestMigrator_ImportWaitExpired(t *testing.T) { srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) tmpl := `{"id":"%d","pid":"","text":"test test #1
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}` recs := []string{} for i := 0; i < 5000; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r) req.SetBasicAuth("admin", "password") require.Nil(t, err) resp, err := client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusAccepted, resp.StatusCode) client = &http.Client{Timeout: 10 * time.Second} req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil) req.SetBasicAuth("admin", "password") assert.NoError(t, err) resp, err = client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode) } func TestMigrator_Export(t *testing.T) { srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"test test #1
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"test test #2
","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) // import comments first client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r) require.Nil(t, err) req.SetBasicAuth("admin", "password") resp, err := client.Do(req) require.Nil(t, err) require.Equal(t, http.StatusAccepted, resp.StatusCode) client = &http.Client{Timeout: 10 * time.Second} req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) req.SetBasicAuth("admin", "password") assert.NoError(t, err) resp, err = client.Do(req) assert.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) // check file mode req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil) require.Nil(t, err) req.SetBasicAuth("admin", "password") resp, err = client.Do(req) require.Nil(t, err) require.Equal(t, 200, resp.StatusCode) require.Equal(t, "application/gzip", resp.Header.Get("Content-Type")) ungzReader, err := gzip.NewReader(resp.Body) assert.NoError(t, err) ungzBody, err := ioutil.ReadAll(ungzReader) assert.NoError(t, err) assert.Equal(t, 3, strings.Count(string(ungzBody), "\n")) assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\"")) t.Logf("%s", string(ungzBody)) // check stream mode req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil) require.Nil(t, err) req.SetBasicAuth("admin", "password") resp, err = client.Do(req) require.Nil(t, err) require.Equal(t, 200, resp.StatusCode) require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type")) body, err := ioutil.ReadAll(resp.Body) assert.NoError(t, err) assert.Equal(t, 3, strings.Count(string(body), "\n")) assert.Equal(t, 2, strings.Count(string(body), "\"text\"")) t.Logf("%s", string(body)) req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil) require.Nil(t, err) resp, err = client.Do(req) require.Nil(t, err) require.Equal(t, http.StatusUnauthorized, resp.StatusCode) } func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com") dataStore := &service.DataStore{Interface: b, AdminStore: adminStore} svc = &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, NativeImporter: &migrator.Native{DataStore: dataStore}, NativeExporter: &migrator.Native{DataStore: dataStore}, Cache: &cache.Nop{}, KeyStore: adminStore, } a := auth.NewService(auth.Opts{ AdminPasswd: "password", SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }), Issuer: "test", }) am := a.Middleware() routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly)) ts = httptest.NewServer(routes) return svc, dataStore, ts } func cleanupImportSrv(_ *Migrator, ts *httptest.Server) { ts.Close() _ = os.Remove(testDb) } var xmlTestWP = `
Meet Grokknomel?]]>