From e3d6869cb052abef9b1fd575c6e9fc15800cca7a Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 5 Mar 2019 22:32:34 -0600 Subject: [PATCH] common teardown for migrator tests, simplify WP import test --- backend/app/rest/api/migrator_test.go | 125 ++++++++-------------- backend/app/rest/api/rest_private_test.go | 5 +- backend/app/rest/api/rest_test.go | 7 +- 3 files changed, 55 insertions(+), 82 deletions(-) diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index cc32c556..68b6e57a 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -3,7 +3,6 @@ package api import ( "bytes" "compress/gzip" - "encoding/json" "fmt" "io" "io/ioutil" @@ -15,7 +14,7 @@ import ( "testing" "time" - bolt "github.com/coreos/bbolt" + "github.com/coreos/bbolt" "github.com/go-chi/chi" "github.com/go-pkgz/auth" "github.com/go-pkgz/auth/token" @@ -24,16 +23,14 @@ import ( "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/migrator" - "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) func TestMigrator_Import(t *testing.T) { - srv, _, ts := prepImportSrv(t) - assert.NotNil(t, srv) - defer cleanupImportSrv(srv, ts) + ts, teardown := prepImportSrv(t) + defer teardown() r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) @@ -50,19 +47,12 @@ func TestMigrator_Import(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) - client = &http.Client{Timeout: 10 * time.Second} - req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) - req.SetBasicAuth("admin", "password") - assert.NoError(t, err) - resp, err = client.Do(req) - require.NoError(t, err) - assert.Equal(t, 200, resp.StatusCode) + waitForImportCompletion(t, ts) } func TestMigrator_ImportForm(t *testing.T) { - srv, _, ts := prepImportSrv(t) - assert.NotNil(t, srv) - defer cleanupImportSrv(srv, ts) + ts, teardown := prepImportSrv(t) + defer teardown() r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) @@ -85,18 +75,12 @@ func TestMigrator_ImportForm(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) - client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) - req.SetBasicAuth("admin", "password") - assert.NoError(t, err) - resp, err = client.Do(req) - require.NoError(t, err) - assert.Equal(t, 200, resp.StatusCode) + waitForImportCompletion(t, ts) } + func TestMigrator_ImportFromWP(t *testing.T) { - srv, ds, ts := prepImportSrv(t) - assert.NotNil(t, srv) - defer cleanupImportSrv(srv, ts) + ts, teardown := prepImportSrv(t) + defer teardown() r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1)) @@ -113,34 +97,12 @@ func TestMigrator_ImportFromWP(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b)) - client = &http.Client{Timeout: 10 * time.Second} - req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) - req.SetBasicAuth("admin", "password") - assert.NoError(t, err) - resp, err = client.Do(req) - assert.NoError(t, err) - assert.Equal(t, 200, resp.StatusCode) - - assert.NoError(t, ds.Interface.Close()) - - tsAccess, _, teardownAccess := startupT(t) - defer teardownAccess() - - res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t") - require.Equal(t, 200, code) - comments := []store.Comment{} - err = json.Unmarshal([]byte(res), &comments) - assert.Nil(t, err) - assert.Equal(t, 3, len(comments), "should have 3 comments") - t.Logf("%+v", comments) - assert.Equal(t, "

Looks like http://releases.rancher.com/os/latest is no longer hosted - installs using this base-url are failing.

\n\n

I switched to Github with success:

\n\n
set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n
\n\n

Thanks for the article!

\n", - comments[0].Text) + waitForImportCompletion(t, ts) } func TestMigrator_ImportRejected(t *testing.T) { - srv, _, ts := prepImportSrv(t) - assert.NotNil(t, srv) - defer cleanupImportSrv(srv, ts) + ts, teardown := prepImportSrv(t) + defer teardown() r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) @@ -154,13 +116,12 @@ func TestMigrator_ImportRejected(t *testing.T) { } func TestMigrator_ImportDouble(t *testing.T) { - srv, _, ts := prepImportSrv(t) - assert.NotNil(t, srv) - defer cleanupImportSrv(srv, ts) + ts, teardown := prepImportSrv(t) + defer teardown() tmpl := `{"id":"%d","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}` recs := []string{} - for i := 0; i < 5000; i++ { + for i := 0; i < 150; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records @@ -179,17 +140,16 @@ func TestMigrator_ImportDouble(t *testing.T) { resp, err = client.Do(req) assert.Nil(t, err) assert.Equal(t, http.StatusConflict, resp.StatusCode) - + waitForImportCompletion(t, ts) } func TestMigrator_ImportWaitExpired(t *testing.T) { - srv, _, ts := prepImportSrv(t) - assert.NotNil(t, srv) - defer cleanupImportSrv(srv, ts) + ts, teardown := prepImportSrv(t) + defer teardown() tmpl := `{"id":"%d","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}` recs := []string{} - for i := 0; i < 5000; i++ { + for i := 0; i < 150; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records @@ -208,12 +168,13 @@ func TestMigrator_ImportWaitExpired(t *testing.T) { resp, err = client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode) + + waitForImportCompletion(t, ts) } func TestMigrator_Export(t *testing.T) { - srv, _, ts := prepImportSrv(t) - assert.NotNil(t, srv) - defer cleanupImportSrv(srv, ts) + ts, teardown := prepImportSrv(t) + defer teardown() r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) @@ -226,13 +187,7 @@ func TestMigrator_Export(t *testing.T) { resp, err := client.Do(req) require.Nil(t, err) require.Equal(t, http.StatusAccepted, resp.StatusCode) - client = &http.Client{Timeout: 10 * time.Second} - req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) - req.SetBasicAuth("admin", "password") - assert.NoError(t, err) - resp, err = client.Do(req) - assert.NoError(t, err) - assert.Equal(t, 200, resp.StatusCode) + waitForImportCompletion(t, ts) // check file mode req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil) @@ -273,12 +228,26 @@ func TestMigrator_Export(t *testing.T) { require.Equal(t, http.StatusUnauthorized, resp.StatusCode) } -func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) { +func waitForImportCompletion(t *testing.T, ts *httptest.Server) { + client := &http.Client{Timeout: 10 * time.Second} + req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil) + req.SetBasicAuth("admin", "password") + assert.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + b, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b)) +} + +func prepImportSrv(t *testing.T) (ts *httptest.Server, teardown func()) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com") dataStore := &service.DataStore{Interface: b, AdminStore: adminStore} - svc = &Migrator{ + svc := &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, NativeImporter: &migrator.Native{DataStore: dataStore}, @@ -296,12 +265,12 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *http am := a.Middleware() routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly)) ts = httptest.NewServer(routes) - return svc, dataStore, ts -} - -func cleanupImportSrv(_ *Migrator, ts *httptest.Server) { - ts.Close() - _ = os.Remove(testDb) + teardown = func() { + ts.Close() + require.NoError(t, b.Close()) + _ = os.Remove(testDb) + } + return ts, teardown } var xmlTestWP = ` diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index 2f14461b..6131bfb0 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -432,13 +432,12 @@ func TestRest_UserAllDataManyComments(t *testing.T) { c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)} - for i := 0; i < 478; i++ { + for i := 0; i < 51; i++ { c.ID = fmt.Sprintf("id-%03d", i) c.Timestamp = c.Timestamp.Add(time.Second) _, err := srv.DataService.Create(c) require.Nil(t, err) } - client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil) require.Nil(t, err) @@ -454,7 +453,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) { assert.NoError(t, err) assert.True(t, strings.HasPrefix(string(ungzBody), `{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`)) - assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside") + assert.Equal(t, 51, strings.Count(string(ungzBody), `"text":`), "51 comments inside") } func TestRest_DeleteMe(t *testing.T) { diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 26516247..bf3c43b6 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -176,6 +176,11 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) { } func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { + + os.Remove(testDb) + os.Remove(testHTML) + os.RemoveAll("/tmp/ava-remark42") + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) @@ -226,7 +231,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { teardown = func() { ts.Close() - srv.DataService.Close() + require.NoError(t, srv.DataService.Close()) os.Remove(testDb) os.Remove(testHTML) os.RemoveAll("/tmp/ava-remark42")