mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix user data export

Browse Source
This commit is contained in:
Umputun
2018-06-03 01:21:20 -05:00
parent 194ece1fa0
commit f751e4b1fe
6 changed files with 109 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1
View File
@@ -357,6 +357,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
* `GET /api/v1/user` - get user info, _auth required_
* `PUT /api/v1/vote/{id}?site=site-id&url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decrease. _auth required_
* `GET /api/v1/userdata?site=site-id` - export all user data to gz stream _auth required_
* `POST /api/v1/deleteme?site=site-id` - request deletion of user data. _auth required_
* `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
```go
app/rest/api/rest_private.go
+4 -3
View File
@@ -193,8 +193,9 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
return
}
for i := 0; i < 1000; i++ {
comments, err := s.DataService.User(siteID, user.ID, 1000, i*1000)
// get comments in 100 in each paginated request
for i := 0; i < 100; i++ {
comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't write user comments")
return
@@ -208,7 +209,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't write user comment")
return
}
if len(comments) != 1000 {
if len(comments) != 100 {
break
}
}
app/rest/api/rest_private_test.go
+86
View File
@@ -1,6 +1,7 @@
package api
import (
"compress/gzip"
"encoding/json"
"fmt"
"io/ioutil"
@@ -236,6 +237,84 @@ func TestRest_Vote(t *testing.T) {
assert.Equal(t, map[string]bool{}, cr.Votes)
}
func TestRest_UserAllData(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
// write 3 comments
user := store.User{ID: "dev", Name: "user name 1"}
c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
_, err := srv.DataService.Create(c1)
require.Nil(t, err, "%+v", err)
_, err = srv.DataService.Create(c2)
require.Nil(t, err)
_, err = srv.DataService.Create(c3)
require.Nil(t, err)
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
require.Nil(t, err)
req = withBasicAuth(req, "dev", "password")
resp, err := client.Do(req)
require.Nil(t, err)
require.Equal(t, 200, resp.StatusCode)
require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
ungzReader, err := gzip.NewReader(resp.Body)
assert.NoError(t, err)
ungzBody, err := ioutil.ReadAll(ungzReader)
assert.NoError(t, err)
assert.True(t, strings.HasPrefix(string(ungzBody),
`{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}[`))
assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
t.Logf("%s", string(ungzBody))
req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
require.Nil(t, err)
resp, err = client.Do(req)
require.Nil(t, err)
require.Equal(t, 401, resp.StatusCode)
}
func TestRest_UserAllDataManyComments(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
user := store.User{ID: "dev", Name: "user name 1"}
c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
for i := 0; i < 478; i++ {
c.ID = fmt.Sprintf("id-%03d", i)
c.Timestamp = c.Timestamp.Add(time.Second)
_, err := srv.DataService.Create(c)
require.Nil(t, err)
}
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
require.Nil(t, err)
req = withBasicAuth(req, "dev", "password")
resp, err := client.Do(req)
require.Nil(t, err)
require.Equal(t, 200, resp.StatusCode)
require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
ungzReader, err := gzip.NewReader(resp.Body)
assert.NoError(t, err)
ungzBody, err := ioutil.ReadAll(ungzReader)
assert.NoError(t, err)
assert.True(t, strings.HasPrefix(string(ungzBody),
`{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}[`))
assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
}
func TestRest_DeleteMe(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
@@ -247,6 +326,7 @@ func TestRest_DeleteMe(t *testing.T) {
req = withBasicAuth(req, "dev", "password")
resp, err := client.Do(req)
assert.Nil(t, err)
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
@@ -260,4 +340,10 @@ func TestRest_DeleteMe(t *testing.T) {
claims, err := srv.Authenticator.JWTService.Parse(token)
assert.Nil(t, err)
assert.Equal(t, "dev", claims.User.ID)
req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
assert.Nil(t, err)
resp, err = client.Do(req)
assert.Nil(t, err)
assert.Equal(t, 401, resp.StatusCode)
}
app/rest/api/rest_public_test.go
-40
View File
@@ -1,7 +1,6 @@
package api
import (
"compress/gzip"
"encoding/json"
"fmt"
"io/ioutil"
@@ -424,42 +423,3 @@ func TestRest_Info(t *testing.T) {
_, code = get(t, ts.URL+"/api/v1/info?site=radio-t-no&url=https://radio-t.com/blah-no")
assert.Equal(t, 400, code)
}
func TestRest_UserAllData(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
// write 3 comments
user := store.User{ID: "dev", Name: "user name 1"}
c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
_, err := srv.DataService.Create(c1)
require.Nil(t, err, "%+v", err)
_, err = srv.DataService.Create(c2)
require.Nil(t, err)
_, err = srv.DataService.Create(c3)
require.Nil(t, err)
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
require.Nil(t, err)
req = withBasicAuth(req, "dev", "password")
resp, err := client.Do(req)
require.Nil(t, err)
require.Equal(t, 200, resp.StatusCode)
require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
ungzReader, err := gzip.NewReader(resp.Body)
assert.NoError(t, err)
ungzBody, err := ioutil.ReadAll(ungzReader)
assert.NoError(t, err)
assert.True(t, strings.HasPrefix(string(ungzBody),
`{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}[`))
assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
t.Logf("%s", string(ungzBody))
}
app/store/engine/bolt_accessor.go
+1 -1
View File
@@ -39,7 +39,7 @@ const (
// limits
lastLimit = 1000
userLimit = 50
userLimit = 500
)
const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
app/store/engine/bolt_accessor_test.go
+17 -11
View File
@@ -258,8 +258,8 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
User: store.User{ID: "user1", Name: "user name"},
}
// write 50 comments
for i := 0; i < 50; i++ {
// write 200 comments
for i := 0; i < 200; i++ {
c.ID = fmt.Sprintf("id-%d", i)
c.Text = fmt.Sprintf("text #%d", i)
c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
@@ -267,29 +267,35 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
require.Nil(t, err)
}
// get all comments
res, err := b.User("radio-t", "user1", 0, 0)
assert.Nil(t, err)
assert.Equal(t, 200, len(res))
assert.Equal(t, "id-199", res[0].ID)
// seek 0, 5 comments
res, err := b.User("radio-t", "user1", 5, 0)
res, err = b.User("radio-t", "user1", 5, 0)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-49", res[0].ID)
assert.Equal(t, "id-45", res[4].ID)
assert.Equal(t, "id-199", res[0].ID)
assert.Equal(t, "id-195", res[4].ID)
// seek 10, 3 comments
res, err = b.User("radio-t", "user1", 3, 10)
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, "id-39", res[0].ID)
assert.Equal(t, "id-37", res[2].ID)
assert.Equal(t, "id-189", res[0].ID)
assert.Equal(t, "id-187", res[2].ID)
// seek 45, ask 10 comments
res, err = b.User("radio-t", "user1", 10, 45)
// seek 195, ask 10 comments
res, err = b.User("radio-t", "user1", 10, 195)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-4", res[0].ID)
assert.Equal(t, "id-0", res[4].ID)
// seek 55, ask 10 comments
res, err = b.User("radio-t", "user1", 10, 55)
// seek 255, ask 10 comments
res, err = b.User("radio-t", "user1", 10, 255)
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}