From 4ba7fb20ae1e3b326afab2dbf586a33c4bd500c9 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 1 Jun 2018 23:27:30 -0500 Subject: [PATCH] add rest to export all user-related data #47 --- README.md | 1 + app/rest/api/rest.go | 52 +++++++++++++++++++++++++++++++++++++++ app/rest/api/rest_test.go | 40 ++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) diff --git a/README.md b/README.md index 48d9c76a..d90f7b66 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i ``` * `GET /api/v1/user` - get user info, _auth required_ * `PUT /api/v1/vote/{id}?site=site-id&url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decrease. _auth required_ +* `GET /api/v1/userdata?site=site-id` - export all user data to gz stream _auth required_ * `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site ```go diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 821ff33c..60c707a8 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -2,6 +2,7 @@ package api import ( "bytes" + "compress/gzip" "context" "crypto/sha1" "encoding/base64" @@ -159,6 +160,7 @@ func (s *Rest) routes() chi.Router { rauth.Put("/comment/{id}", s.updateCommentCtrl) rauth.Get("/user", s.userInfoCtrl) rauth.Put("/vote/{id}", s.voteCtrl) + rauth.Get("/userdata", s.userAllDataCtrl) // admin routes, admin users only rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly, Logger(nil, LogAll))) @@ -605,6 +607,56 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score}) } +// GET /userdata?site=siteID - exports all data about the user as a json fragments +func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site") + user, err := rest.GetUserInfo(r) + if err != nil { + rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") + return + } + userB, err := json.Marshal(&user) + if err != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info") + return + } + + exportFile := fmt.Sprintf("%s-%s-%s.json.gz", siteID, user.ID, time.Now().Format("20060102")) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Disposition", "attachment;filename="+exportFile) + gzWriter := gzip.NewWriter(w) + defer func() { + if e := gzWriter.Close(); e != nil { + log.Printf("[WARN] can't close gzip writer, %s", e) + } + }() + + if _, e := gzWriter.Write(userB); e != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't write user info") + return + } + + for i := 0; i < 1000; i++ { + comments, err := s.DataService.User(siteID, user.ID, 1000, i*1000) + if err != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't write user comments") + return + } + b, err := json.Marshal(comments) + if err != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments") + return + } + if _, e := gzWriter.Write(b); e != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't write user comment") + return + } + if len(comments) != 1000 { + break + } + } +} + // serves static files from /web func addFileServer(r chi.Router, path string, root http.FileSystem) { log.Printf("[INFO] run file server for %s, path %s", root, path) diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go index 34ec3ce5..39e1127e 100644 --- a/app/rest/api/rest_test.go +++ b/app/rest/api/rest_test.go @@ -2,6 +2,7 @@ package api import ( "bytes" + "compress/gzip" "encoding/base64" "encoding/json" "fmt" @@ -659,6 +660,45 @@ func TestRest_Info(t *testing.T) { assert.Equal(t, 400, code) } +func TestRest_UserAllData(t *testing.T) { + srv, ts := prep(t) + assert.NotNil(t, srv) + defer cleanup(ts) + + // write 3 comments + user := store.User{ID: "dev", Name: "user name 1"} + c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)} + c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)} + c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)} + _, err := srv.DataService.Create(c1) + require.Nil(t, err, "%+v", err) + _, err = srv.DataService.Create(c2) + require.Nil(t, err) + _, err = srv.DataService.Create(c3) + require.Nil(t, err) + + client := &http.Client{Timeout: 1 * time.Second} + req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil) + require.Nil(t, err) + req = withBasicAuth(req, "dev", "password") + resp, err := client.Do(req) + require.Nil(t, err) + require.Equal(t, 200, resp.StatusCode) + require.Equal(t, "application/gzip", resp.Header.Get("Content-Type")) + + ungzReader, err := gzip.NewReader(resp.Body) + assert.NoError(t, err) + ungzBody, err := ioutil.ReadAll(ungzReader) + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(string(ungzBody), + `{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}[`)) + assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside") + t.Logf("%s", string(ungzBody)) +} + func TestRest_FileServer(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv)