add rest to export all user-related data #47

README.md

@@ -357,6 +357,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
   ```
 * `GET /api/v1/user` - get user info, _auth required_
 * `PUT /api/v1/vote/{id}?site=site-id&url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decrease. _auth required_
+* `GET /api/v1/userdata?site=site-id` - export all user data to gz stream  _auth required_
 * `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
 
   ```go

app/rest/api/rest.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/sha1"
 	"encoding/base64"
@@ -159,6 +160,7 @@ func (s *Rest) routes() chi.Router {
 			rauth.Put("/comment/{id}", s.updateCommentCtrl)
 			rauth.Get("/user", s.userInfoCtrl)
 			rauth.Put("/vote/{id}", s.voteCtrl)
+			rauth.Get("/userdata", s.userAllDataCtrl)
 
 			// admin routes, admin users only
 			rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly, Logger(nil, LogAll)))
@@ -605,6 +607,56 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score})
 }
 
+// GET /userdata?site=siteID - exports all data about the user as a json fragments
+func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	user, err := rest.GetUserInfo(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
+		return
+	}
+	userB, err := json.Marshal(&user)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
+		return
+	}
+
+	exportFile := fmt.Sprintf("%s-%s-%s.json.gz", siteID, user.ID, time.Now().Format("20060102"))
+	w.Header().Set("Content-Type", "application/gzip")
+	w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
+	gzWriter := gzip.NewWriter(w)
+	defer func() {
+		if e := gzWriter.Close(); e != nil {
+			log.Printf("[WARN] can't close gzip writer, %s", e)
+		}
+	}()
+
+	if _, e := gzWriter.Write(userB); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't write user info")
+		return
+	}
+
+	for i := 0; i < 1000; i++ {
+		comments, err := s.DataService.User(siteID, user.ID, 1000, i*1000)
+		if err != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't write user comments")
+			return
+		}
+		b, err := json.Marshal(comments)
+		if err != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments")
+			return
+		}
+		if _, e := gzWriter.Write(b); e != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't write user comment")
+			return
+		}
+		if len(comments) != 1000 {
+			break
+		}
+	}
+}
+
 // serves static files from /web
 func addFileServer(r chi.Router, path string, root http.FileSystem) {
 	log.Printf("[INFO] run file server for %s, path %s", root, path)

app/rest/api/rest_test.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"bytes"
+	"compress/gzip"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -659,6 +660,45 @@ func TestRest_Info(t *testing.T) {
 	assert.Equal(t, 400, code)
 }
 
+func TestRest_UserAllData(t *testing.T) {
+	srv, ts := prep(t)
+	assert.NotNil(t, srv)
+	defer cleanup(ts)
+
+	// write 3 comments
+	user := store.User{ID: "dev", Name: "user name 1"}
+	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
+	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
+	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
+	_, err := srv.DataService.Create(c1)
+	require.Nil(t, err, "%+v", err)
+	_, err = srv.DataService.Create(c2)
+	require.Nil(t, err)
+	_, err = srv.DataService.Create(c3)
+	require.Nil(t, err)
+
+	client := &http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
+	require.Nil(t, err)
+	req = withBasicAuth(req, "dev", "password")
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, 200, resp.StatusCode)
+	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
+
+	ungzReader, err := gzip.NewReader(resp.Body)
+	assert.NoError(t, err)
+	ungzBody, err := ioutil.ReadAll(ungzReader)
+	assert.NoError(t, err)
+	assert.True(t, strings.HasPrefix(string(ungzBody),
+		`{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}[`))
+	assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
+	t.Logf("%s", string(ungzBody))
+}
+
 func TestRest_FileServer(t *testing.T) {
 	srv, ts := prep(t)
 	assert.NotNil(t, srv)