implement user details storage (#469)

* implement (strings) user details storage

* add rpc user details implementation

* return error from getUserDetail, rewrite tests to table tests

* make UserDetails store UserDetailEntry instead of strings

* update comment about user_details

* fix confusing return

* add user details support for memory store

* add engine.UserDetailEntry to service.UserMetaData

* add ListDetails support to memory storage

* add user details support to native migrator, ListDetails func to storage

* go mod tidy for memory storage

* increase memory storage test coverage, fix tests naming

* add ListDetails tests to memory storage

* add engine.ListDetails and  service.[Set]Metas tests

* change Fprintf to Fprint (triggered by explicitly ignoring error)

* remove Delete from engine.UserDetail, implement list via same method

* adjust service.Metas to new engine.UserDetails signature

* introduce engine.UserDetail("all") consonant

* fix Meta user detail retrieval

* extend store implementations Delete method with UserDetail deletion

* make UserDetail test answer order-independent

* fix flaky test check in TestMemData_FlagListBlocked

* delete user details alongside with comments on deleteme request

* add tests to UserDetail store.Delete implementations

* clarify engine module user details consonants names

* update comments to reflect current state of code

* check for value absence instead of it's length

* revert unneeded code change

* add extensive commentary on UserDetail return type

* remove unused check condition

* clarify UserDetail tests to be truly stateless

* add clarifying comment for pre-table test
This commit is contained in:
Dmitry Verkhoturov
2019-11-22 02:24:26 -06:00
committed by Umputun
parent 773da16649
commit ddd466ec41
13 changed files with 659 additions and 56 deletions
+129 -5
View File
@@ -41,6 +41,7 @@ type metaUser struct {
Verified bool
Blocked bool
BlockedUntil time.Time
Details engine.UserDetailEntry
}
// NewMemData makes in-memory engine.
@@ -280,17 +281,51 @@ func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err erro
return nil, errors.Errorf("flag %s not listable", req.Flag)
}
// Delete post(s), user, comment, or everything
// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
switch req.Detail {
case engine.UserEmail:
if req.UserID == "" {
return nil, errors.New("userid cannot be empty in request for single detail")
}
m.Lock()
defer m.Unlock()
if req.Update == "" { // read detail value, no update requested
return m.getUserDetail(req)
}
return m.setUserDetail(req)
case engine.AllUserDetails:
// list of all details returned in case request is a read request
// (Update is not set) and does not have UserID or Detail set
if req.Update == "" && req.UserID == "" { // read list of all details
m.Lock()
defer m.Unlock()
return m.listDetails(req.Locator)
}
return nil, errors.New("unsupported request with userdetail all")
default:
return nil, errors.Errorf("unsupported detail %q", req.Detail)
}
}
// Delete post(s), user, comment, user details, or everything
func (m *MemData) Delete(req engine.DeleteRequest) error {
m.Lock()
defer m.Unlock()
switch {
case req.Locator.URL != "" && req.CommentID != "": // delete comment
case req.UserDetail != "": // delete user detail
return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "": // delete user
case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
return c.User.ID == req.UserID && !c.Deleted
})
@@ -299,9 +334,9 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
return e
}
}
return nil
return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "": // delete site
case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
if _, ok := m.posts[req.Locator.SiteID]; !ok {
return errors.New("not found")
}
@@ -401,6 +436,95 @@ func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
return status, errors.Wrapf(err, "failed to set flag %+v", req)
}
// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
// as an only element of the slice.
func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
if meta, ok := m.metaUsers[req.UserID]; ok {
if meta.SiteID != req.Locator.SiteID {
return []engine.UserDetailEntry{}, nil
}
switch req.Detail {
case engine.UserEmail:
return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
}
}
return []engine.UserDetailEntry{}, nil
}
// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
// element of the slice in case of success
func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
var entry metaUser
if meta, ok := m.metaUsers[req.UserID]; ok {
if meta.SiteID != req.Locator.SiteID {
return []engine.UserDetailEntry{}, nil
}
entry = meta
}
if entry == (metaUser{}) {
entry = metaUser{
UserID: req.UserID,
SiteID: req.Locator.SiteID,
Details: engine.UserDetailEntry{UserID: req.UserID},
}
}
switch req.Detail {
case engine.UserEmail:
entry.Details.Email = req.Update
m.metaUsers[req.UserID] = entry
return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
}
return []engine.UserDetailEntry{}, nil
}
// listDetails lists all available users details for given siteID
func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
var res []engine.UserDetailEntry
for _, u := range m.metaUsers {
if u.SiteID == loc.SiteID {
res = append(res, u.Details)
}
}
return res, nil
}
// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
// deletion of the absent entry doesn't produce error.
// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
var entry metaUser
if meta, ok := m.metaUsers[userID]; ok {
if meta.SiteID != locator.SiteID {
return nil
}
entry = meta
}
if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
// absent entry means that we should not do anything
return nil
}
switch userDetail {
case engine.UserEmail:
entry.Details.Email = ""
case engine.AllUserDetails:
entry.Details = engine.UserDetailEntry{UserID: userID}
}
if entry.Details == (engine.UserDetailEntry{UserID: userID}) {
// no user details are stored, empty details entry altogether
entry.Details = engine.UserDetailEntry{}
}
m.metaUsers[userID] = entry
return nil
}
func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
return c.Locator == loc && c.ID == commentID
@@ -658,6 +658,43 @@ func TestMemData_DeleteAll(t *testing.T) {
assert.Equal(t, 0, len(comments), "nothing left")
}
func TestMemData_DeleteUserDetail(t *testing.T) {
var (
createUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
readUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
emailSet = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
emailUnset = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
)
b := prepMem(t)
var testData = []struct {
delReq engine.DeleteRequest
detailReq engine.UserDetailRequest
expected []engine.UserDetailEntry
}{
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
detailReq: createUser, expected: emailSet},
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
detailReq: readUser, expected: emailSet},
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
detailReq: readUser, expected: emailUnset},
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
detailReq: createUser, expected: emailSet},
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
detailReq: readUser, expected: emailUnset},
}
for i, x := range testData {
err := b.Delete(x.delReq)
require.NoError(t, err, "delete request #%d error", i)
val, err := b.UserDetail(x.detailReq)
require.NoError(t, err, "user request #%d error", i)
require.Equal(t, x.expected, val, "user request #%d result", i)
}
}
func TestMemAdmin_DeleteUserHard(t *testing.T) {
b := prepMem(t)
err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+24 -11
View File
@@ -28,16 +28,17 @@ func NewRPC(e engine.Interface, a admin.Store, r *jrpc.Server) *RPC {
func (s *RPC) addHandlers() {
// data store handlers
s.Group("store", jrpc.HandlersGroup{
"create": s.createHndl,
"find": s.findHndl,
"get": s.getHndl,
"update": s.updateHndl,
"count": s.countHndl,
"info": s.infoHndl,
"flag": s.flagHndl,
"list_flags": s.listFlagsHndl,
"delete": s.deleteHndl,
"close": s.closeHndl,
"create": s.createHndl,
"find": s.findHndl,
"get": s.getHndl,
"update": s.updateHndl,
"count": s.countHndl,
"info": s.infoHndl,
"flag": s.flagHndl,
"list_flags": s.listFlagsHndl,
"user_detail": s.userDetailHndl,
"delete": s.deleteHndl,
"close": s.closeHndl,
})
// admin store handlers
@@ -129,7 +130,19 @@ func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response
return jrpc.EncodeResponse(id, flags, err)
}
// deleteHndl delete post(s), user, comment, or everything
// userDetailHndl sets or gets single detail value, or gets all details for requested site.
// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
req := engine.UserDetailRequest{}
if err := json.Unmarshal(params, &req); err != nil {
return jrpc.Response{Error: err.Error()}
}
value, err := s.eng.UserDetail(req)
return jrpc.EncodeResponse(id, value, err)
}
// deleteHndl delete post(s), user, comment, user details, or everything
func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
req := engine.DeleteRequest{}
if err := json.Unmarshal(params, &req); err != nil {
@@ -223,6 +223,59 @@ func TestRPC_listFlagsHndl(t *testing.T) {
assert.Equal(t, []interface{}{"u1"}, flags)
}
func TestRPC_userDetailHndl(t *testing.T) {
_, port, teardown := prepTestStore(t)
defer teardown()
api := fmt.Sprintf("http://localhost:%d/test", port)
re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
// add to entries to DB before we start
result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
assert.NoError(t, err, "No error inserting entry expected")
assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
assert.NoError(t, err, "No error inserting entry expected")
assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
// try to change existing entry with wrong SiteID
result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
// stateless tests without changing the state we set up before
var testData = []struct {
req engine.UserDetailRequest
error string
expected []engine.UserDetailEntry
}{
{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
expected: []engine.UserDetailEntry{}},
{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
expected: []engine.UserDetailEntry{}},
{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
error: `userid cannot be empty in request for single detail`},
{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
error: `unsupported detail "bad"`},
{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
error: `unsupported request with userdetail all`},
{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
}
for i, x := range testData {
result, err := re.UserDetail(x.req)
if x.error != "" {
assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
} else {
assert.NoError(t, err, "Error is not expected expected for case %d", i)
}
assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
}
}
func TestRPC_deleteHndl(t *testing.T) {
_, port, teardown := prepTestStore(t)
defer teardown()
+1 -1
View File
@@ -89,7 +89,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, ucomments[0].User)
}
// GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
// GET /deleteme?token=jwt - delete all user comments and details by user's request. Gets info about deleted used from provided token
// request made GET to allow direct click from the email sent by user
func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
+185 -18
View File
@@ -16,12 +16,13 @@ import (
)
// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
// there are 5 types of top-level buckets:
// there are 6 types of top-level buckets:
// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
// value is not full comment but a reference combined from post-url+commentID
// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
// is a nested bucket named userID with kv as ts:reference
// - users details in "user_details" bucket. Key is userID, value - UserDetailEntry
// - blocking info sits in "block" bucket. Key is userID, value - ts
// - counts per post to keep number of comments. Key is post url, value - count
// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
@@ -31,13 +32,14 @@ type BoltDB struct {
const (
// top level buckets
postsBucketName = "posts"
lastBucketName = "last"
userBucketName = "users"
blocksBucketName = "block"
infoBucketName = "info"
readonlyBucketName = "readonly"
verifiedBucketName = "verified"
postsBucketName = "posts"
lastBucketName = "last"
userBucketName = "users"
userDetailsBucketName = "user_details"
blocksBucketName = "block"
infoBucketName = "info"
readonlyBucketName = "readonly"
verifiedBucketName = "verified"
tsNano = "2006-01-02T15:04:05.000000000Z07:00"
)
@@ -59,8 +61,8 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
}
// make top-level buckets
topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, infoBucketName,
readonlyBucketName, verifiedBucketName}
topBuckets := []string{postsBucketName, lastBucketName, userBucketName, userDetailsBucketName,
blocksBucketName, infoBucketName, readonlyBucketName, verifiedBucketName}
err = db.Update(func(tx *bolt.Tx) error {
for _, bktName := range topBuckets {
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
@@ -204,6 +206,33 @@ func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) {
return b.setFlag(req)
}
// UserDetail sets or gets single detail value, or gets all details for requested site.
// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
// and all site's details listing under the same function (and not to extend interface by two separate functions).
func (b *BoltDB) UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) {
switch req.Detail {
case UserEmail:
if req.UserID == "" {
return nil, errors.New("userid cannot be empty in request for single detail")
}
if req.Update == "" { // read detail value, no update requested
return b.getUserDetail(req)
}
return b.setUserDetail(req)
case AllUserDetails:
// list of all details returned in case request is a read request
// (Update is not set) and does not have UserID
if req.Update == "" && req.UserID == "" { // read list of all details
return b.listDetails(req.Locator)
}
return nil, errors.New("unsupported request with userdetail all")
default:
return nil, errors.Errorf("unsupported detail %q", req.Detail)
}
}
// Update for locator.URL with mutable part of comment
func (b *BoltDB) Update(comment store.Comment) error {
@@ -369,7 +398,7 @@ func (b *BoltDB) ListFlags(req FlagRequest) (res []interface{}, err error) {
return nil, errors.Errorf("flag %s not listable", req.Flag)
}
// Delete post(s) by id or by userID
// Delete post(s), user, comment, user details, or everything
func (b *BoltDB) Delete(req DeleteRequest) error {
bdb, e := b.db(req.Locator.SiteID)
@@ -378,11 +407,13 @@ func (b *BoltDB) Delete(req DeleteRequest) error {
}
switch {
case req.Locator.URL != "" && req.CommentID != "":
case req.UserDetail != "": // delete user detail
return b.deleteUserDetail(bdb, req.UserID, req.UserDetail)
case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode)
case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "":
case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
return b.deleteUser(bdb, req.Locator.SiteID, req.UserID, req.DeleteMode)
case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "":
case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
return b.deleteAll(bdb, req.Locator.SiteID)
}
@@ -617,6 +648,142 @@ func (b *BoltDB) flagBucket(tx *bolt.Tx, flag Flag) (bkt *bolt.Bucket, err error
return bkt, nil
}
// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
// as an only element of the slice.
func (b *BoltDB) getUserDetail(req UserDetailRequest) (result []UserDetailEntry, err error) {
bdb, e := b.db(req.Locator.SiteID)
if e != nil {
return result, e
}
err = bdb.View(func(tx *bolt.Tx) error {
var entry UserDetailEntry
bucket := tx.Bucket([]byte(userDetailsBucketName))
value := bucket.Get([]byte(req.UserID))
// return no error in case of absent entry
if value != nil {
if err := json.Unmarshal(value, &entry); err != nil {
return errors.Wrap(e, "failed to unmarshal entry")
}
switch req.Detail {
case UserEmail:
result = []UserDetailEntry{{UserID: req.UserID, Email: entry.Email}}
}
}
return nil
})
return result, err
}
// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
// element of the slice in case of success
func (b *BoltDB) setUserDetail(req UserDetailRequest) (result []UserDetailEntry, err error) {
bdb, e := b.db(req.Locator.SiteID)
if e != nil {
return result, e
}
var entry UserDetailEntry
err = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(userDetailsBucketName))
value := bucket.Get([]byte(req.UserID))
// return no error in case of absent entry
if value != nil {
if err := json.Unmarshal(value, &entry); err != nil {
return errors.Wrap(e, "failed to unmarshal entry")
}
}
return nil
})
if err != nil {
return result, err
}
if entry.UserID == "" {
// new entry to be created, need to set UserID for it
entry.UserID = req.UserID
}
switch req.Detail {
case UserEmail:
entry.Email = req.Update
}
err = bdb.Update(func(tx *bolt.Tx) error {
err := b.save(tx.Bucket([]byte(userDetailsBucketName)), req.UserID, entry)
return errors.Wrapf(err, "failed to update detail %s for %s in %s", req.Detail, req.UserID, req.Locator.SiteID)
})
return []UserDetailEntry{entry}, err
}
// listDetails lists all available users details for given site
func (b *BoltDB) listDetails(loc store.Locator) (result []UserDetailEntry, err error) {
bdb, e := b.db(loc.SiteID)
if e != nil {
return result, e
}
err = bdb.View(func(tx *bolt.Tx) error {
var entry UserDetailEntry
bucket := tx.Bucket([]byte(userDetailsBucketName))
return bucket.ForEach(func(userID, value []byte) error {
if err := json.Unmarshal(value, &entry); err != nil {
return errors.Wrap(e, "failed to unmarshal entry")
}
result = append(result, entry)
return nil
})
})
return result, err
}
// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry
func (b *BoltDB) deleteUserDetail(bdb *bolt.DB, userID string, userDetail UserDetail) error {
var entry UserDetailEntry
err := bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(userDetailsBucketName))
value := bucket.Get([]byte(userID))
// return no error in case of absent entry
if value != nil {
if err := json.Unmarshal(value, &entry); err != nil {
return errors.Wrap(err, "failed to unmarshal entry")
}
}
return nil
})
if err != nil {
return err
}
if entry == (UserDetailEntry{}) {
// absent entry means that we should not do anything
return nil
}
switch userDetail {
case UserEmail:
entry.Email = ""
case AllUserDetails:
entry = UserDetailEntry{UserID: userID}
}
if entry == (UserDetailEntry{UserID: userID}) {
// if entry doesn't have non-empty details, we should delete it
return bdb.Update(func(tx *bolt.Tx) error {
err := tx.Bucket([]byte(userDetailsBucketName)).Delete([]byte(userID))
return errors.Wrapf(err, "failed to delete user detail %s for %s", userDetail, userID)
})
}
return bdb.Update(func(tx *bolt.Tx) error {
// updated entry is not empty and we need to store it's updated copy
err := b.save(tx.Bucket([]byte(userDetailsBucketName)), userID, entry)
return errors.Wrapf(err, "failed to update detail %s for %s", userDetail, userID)
})
}
func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID string, mode store.DeleteMode) error {
return bdb.Update(func(tx *bolt.Tx) error {
@@ -656,7 +823,7 @@ func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID st
func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
// delete all buckets except blocked users
toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
toDelete := []string{postsBucketName, lastBucketName, userBucketName, userDetailsBucketName, infoBucketName}
// delete top-level buckets
err := bdb.Update(func(tx *bolt.Tx) error {
@@ -675,7 +842,7 @@ func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
}
// deleteUser removes all comments for given user. Everything will be market as deleted
// deleteUser removes all comments and details for given user. Everything will be market as deleted
// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error {
@@ -722,7 +889,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode sto
}
}
// delete user bucket in hard mode
// delete user bucket in hard mode
if mode == store.HardDelete {
err = bdb.Update(func(tx *bolt.Tx) error {
usersBkt := tx.Bucket([]byte(userBucketName))
@@ -743,7 +910,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode sto
return errors.Errorf("unknown user %s", userID)
}
return err
return b.deleteUserDetail(bdb, userID, AllUserDetails)
}
// getPostBucket return bucket with all comments for postURL
+89
View File
@@ -620,6 +620,53 @@ func TestBolt_FlagListBlocked(t *testing.T) {
}
func TestBoltDB_UserDetail(t *testing.T) {
b, teardown := prep(t)
defer teardown()
// add to entries to DB before we start
result, err := b.UserDetail(UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1", Detail: UserEmail, Update: "test@example.com"})
assert.NoError(t, err, "No error inserting entry expected")
assert.ElementsMatch(t, []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
result, err = b.UserDetail(UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u2", Detail: UserEmail, Update: "other@example.com"})
assert.NoError(t, err, "No error inserting entry expected")
assert.ElementsMatch(t, []UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
// stateless tests without changing the state we set up before
var testData = []struct {
req UserDetailRequest
error string
expected []UserDetailEntry
}{
{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1", Detail: UserEmail},
expected: []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
{req: UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: UserEmail},
error: `site "bad" not found`},
{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1xyz", Detail: UserEmail}},
{req: UserDetailRequest{Detail: UserEmail, Update: "new_value"},
error: `userid cannot be empty in request for single detail`},
{req: UserDetailRequest{Detail: UserDetail("bad")},
error: `unsupported detail "bad"`},
{req: UserDetailRequest{Update: "not_relevant", Detail: AllUserDetails},
error: `unsupported request with userdetail all`},
{req: UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, Detail: AllUserDetails},
error: `site "bad" not found`},
{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, Detail: AllUserDetails},
expected: []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
}
for i, x := range testData {
result, err := b.UserDetail(x.req)
if x.error != "" {
assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
} else {
assert.NoError(t, err, "Error is not expected expected for case %d", i)
}
assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
}
}
func TestBolt_DeleteComment(t *testing.T) {
b, teardown := prep(t)
@@ -713,6 +760,48 @@ func TestBolt_DeleteAll(t *testing.T) {
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBolt_DeleteUserDetail(t *testing.T) {
var (
createUser = UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Detail: UserEmail, Update: "value1"}
readUser = UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Detail: UserEmail}
emailSet = []UserDetailEntry{{UserID: "user1", Email: "value1"}}
)
b, teardown := prep(t)
defer teardown()
var testData = []struct {
delReq DeleteRequest
detailReq UserDetailRequest
expected []UserDetailEntry
err string
}{
{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: UserEmail},
detailReq: createUser, expected: emailSet},
{delReq: DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: UserEmail},
detailReq: readUser, expected: emailSet, err: `site "bad" not found`},
{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: UserEmail},
detailReq: readUser},
{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: AllUserDetails},
detailReq: createUser, expected: emailSet},
{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: AllUserDetails},
detailReq: readUser},
}
for i, x := range testData {
err := b.Delete(x.delReq)
if x.err == "" {
require.NoError(t, err, "delete request #%d error", i)
} else {
require.EqualError(t, err, x.err, "delete request #%d error", i)
}
val, err := b.UserDetail(x.detailReq)
require.NoError(t, err, "user request #%d error", i)
require.Equal(t, x.expected, val, "user request #%d result", i)
}
}
func TestBoltAdmin_DeleteUserHard(t *testing.T) {
b, teardown := prep(t)
+36 -10
View File
@@ -16,16 +16,19 @@ import (
// Interface defines methods provided by low-level storage engine
type Interface interface {
Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
Update(comment store.Comment) error // update comment, mutable parts only
Get(req GetRequest) (store.Comment, error) // get comment by id
Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site
Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info
Count(req FindRequest) (int, error) // get count for post or user
Delete(req DeleteRequest) error // delete post(s) by id or by userID
Flag(req FlagRequest) (bool, error) // set and get flags
ListFlags(req FlagRequest) ([]interface{}, error) // get list of flagged keys, like blocked & verified user
Close() error // close storage engine
Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
Update(comment store.Comment) error // update comment, mutable parts only
Get(req GetRequest) (store.Comment, error) // get comment by id
Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site
Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info
Count(req FindRequest) (int, error) // get count for post or user
Delete(req DeleteRequest) error // Delete post(s), user, comment, user details, or everything
Flag(req FlagRequest) (bool, error) // set and get flags
ListFlags(req FlagRequest) ([]interface{}, error) // get list of flagged keys, like blocked & verified user
UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) // sets or gets single detail value, or gets all details for requested site.
// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
// and all site's details listing under the same function (and not to extend interface by two separate functions).
Close() error // close storage engine
}
// GetRequest is the input for Get func
@@ -57,6 +60,7 @@ type DeleteRequest struct {
Locator store.Locator `json:"locator"` // lack of URL means site operation
CommentID string `json:"comment_id,omitempty"`
UserID string `json:"user_id,omitempty"`
UserDetail UserDetail `json:"user_detail,omitempty"`
DeleteMode store.DeleteMode `json:"del_mode"`
}
@@ -79,6 +83,11 @@ const (
Verified = Flag("verified")
Blocked = Flag("blocked")
)
const (
// All possible user details
UserEmail = UserDetail("email")
AllUserDetails = UserDetail("all") // used for listing and deletion requests
)
// FlagRequest is the input for both get/set for flags, like blocked, verified and so on
type FlagRequest struct {
@@ -89,6 +98,23 @@ type FlagRequest struct {
TTL time.Duration `json:"ttl,omitempty"` // ttl for time-sensitive flags only, like blocked for some period
}
// UserDetail defines name of the user detail
type UserDetail string
// UserDetailEntry contains single user details entry
type UserDetailEntry struct {
UserID string `json:"user_id"` // duplicate user's id to use this structure not only embedded but separately
Email string `json:"email,omitempty"` // UserEmail
}
// UserDetailRequest is the input for both get/set for details, like email
type UserDetailRequest struct {
Detail UserDetail `json:"detail"` // detail name
Locator store.Locator `json:"locator"` // post locator
UserID string `json:"user_id"` // user id for get\set
Update string `json:"update,omitempty"` // update value
}
const (
// limits
lastLimit = 1000
+26 -4
View File
@@ -1,10 +1,9 @@
// Code generated by mockery v1.0.0. DO NOT EDIT.
package engine
import (
mock "github.com/stretchr/testify/mock"
store "github.com/umputun/remark/backend/app/store"
)
import mock "github.com/stretchr/testify/mock"
import store "github.com/umputun/remark/backend/app/store"
// MockInterface is an autogenerated mock type for the Interface type
type MockInterface struct {
@@ -205,3 +204,26 @@ func (_m *MockInterface) Update(comment store.Comment) error {
return r0
}
// UserDetail provides a mock function with given fields: req
func (_m *MockInterface) UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) {
ret := _m.Called(req)
var r0 []UserDetailEntry
if rf, ok := ret.Get(0).(func(UserDetailRequest) []UserDetailEntry); ok {
r0 = rf(req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]UserDetailEntry)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(UserDetailRequest) error); ok {
r1 = rf(req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
+13 -1
View File
@@ -82,6 +82,18 @@ func (r *RPC) ListFlags(req FlagRequest) (list []interface{}, err error) {
return list, err
}
// UserDetail sets or gets single detail value, or gets all details for requested site.
// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
// and all site's details listing under the same function (and not to extend interface by two separate functions).
func (r *RPC) UserDetail(req UserDetailRequest) (result []UserDetailEntry, err error) {
resp, err := r.Call("store.user_detail", req)
if err != nil {
return nil, err
}
err = json.Unmarshal(*resp.Result, &result)
return result, err
}
// Count gets comments count by user or site
func (r *RPC) Count(req FindRequest) (count int, err error) {
resp, err := r.Call("store.count", req)
@@ -92,7 +104,7 @@ func (r *RPC) Count(req FindRequest) (count int, err error) {
return count, err
}
// Delete post(s) by id or by userID
// Delete post(s), user, comment, user details, or everything
func (r *RPC) Delete(req DeleteRequest) error {
_, err := r.Call("store.delete", req)
return err
+23 -1
View File
@@ -139,6 +139,28 @@ func TestRemote_ListFlag(t *testing.T) {
assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res)
}
func TestRemote_UserDetail(t *testing.T) {
ts := testServer(t, `{"method":"store.user_detail","params":{"detail":"email","locator":{"url":"http://example.com/url"},"user_id":"username"},"id":1}`, `{"result":[{"user_id":"u1","email":"test_email@example.com"}]}`)
defer ts.Close()
c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
req := UserDetailRequest{Locator: store.Locator{URL: "http://example.com/url"}, UserID: "username", Detail: UserEmail}
res, err := c.UserDetail(req)
assert.NoError(t, err)
assert.Equal(t, []UserDetailEntry{{UserID: "u1", Email: "test_email@example.com"}}, res)
t.Logf("%v %T", res, res)
}
func TestRemote_UserDetailWithErrorResult(t *testing.T) {
ts := testServer(t, `{"method":"store.user_detail","params":{"detail":"email","locator":{"url":"http://example.com/url"},"user_id":"username","update":"new_value@example.com"},"id":1}`, `{"error":"failed"}`)
defer ts.Close()
c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
req := UserDetailRequest{Locator: store.Locator{URL: "http://example.com/url"}, UserID: "username", Detail: UserEmail, Update: "new_value@example.com"}
_, err := c.UserDetail(req)
assert.EqualError(t, err, "failed")
}
func TestRemote_Count(t *testing.T) {
ts := testServer(t, `{"method":"store.count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`)
defer ts.Close()
@@ -173,6 +195,6 @@ func testServer(t *testing.T, req, resp string) *httptest.Server {
require.NoError(t, err)
assert.Equal(t, req, string(body))
t.Logf("req: %s", string(body))
fmt.Fprintf(w, resp)
_, _ = fmt.Fprint(w, resp)
}))
}
+24 -3
View File
@@ -51,14 +51,15 @@ type DataStore struct {
}
}
// UserMetaData keeps info about user flags
// UserMetaData keeps info about user flags and details
type UserMetaData struct {
ID string `json:"id"`
Blocked struct {
Status bool `json:"status"`
Until time.Time `json:"until"`
} `json:"blocked"`
Verified bool `json:"verified"`
Verified bool `json:"verified"`
Details engine.UserDetailEntry `json:"details,omitempty"`
}
// PostMetaData keeps info about post flags
@@ -665,7 +666,7 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
}
}
// set users meta
// set users meta, key is userID
m := map[string]UserMetaData{}
// process blocked users
@@ -698,6 +699,20 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
m[v] = val
}
// process users details
usersDetails, err := s.Engine.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: siteID}, Detail: engine.AllUserDetails})
if err != nil {
return nil, nil, errors.Wrapf(err, "can't get user details for %s", siteID)
}
for _, entry := range usersDetails {
val, ok := m[entry.UserID]
if !ok {
val = UserMetaData{ID: entry.UserID}
}
val.Details = entry
m[entry.UserID] = val
}
for _, u := range m {
umetas = append(umetas, u)
}
@@ -725,6 +740,12 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
if um.Verified {
errs = multierror.Append(errs, s.SetVerified(siteID, um.ID, true))
}
// this code doesn't delete user details in case they are not set in import but present in DB already
if um.Details.Email != "" {
req := engine.UserDetailRequest{Locator: store.Locator{SiteID: siteID}, UserID: um.ID, Detail: engine.UserEmail, Update: um.Details.Email}
_, err := s.Engine.UserDetail(req)
errs = multierror.Append(errs, err)
}
}
return errs.ErrorOrNil()
+19 -2
View File
@@ -718,15 +718,29 @@ func TestService_GetMetas(t *testing.T) {
assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour))
assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
// set email for one existing and one non-existing user
req := engine.UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user2", Detail: engine.UserEmail, Update: "test@example.org"}
value, err := b.Engine.UserDetail(req)
assert.NoError(t, err)
assert.Equal(t, []engine.UserDetailEntry{{UserID: "user2", Email: "test@example.org"}}, value)
req.UserID = "user3"
value, err = b.Engine.UserDetail(req)
assert.NoError(t, err)
assert.Equal(t, []engine.UserDetailEntry{{UserID: "user3", Email: "test@example.org"}}, value)
um, pm, err = b.Metas("radio-t")
require.NoError(t, err)
assert.Equal(t, 2, len(um))
assert.Equal(t, 3, len(um))
assert.Equal(t, "user1", um[0].ID)
assert.Equal(t, true, um[0].Verified)
assert.Equal(t, engine.UserDetailEntry{Email: ""}, um[0].Details)
assert.Equal(t, true, um[0].Blocked.Status)
assert.Equal(t, false, um[1].Verified)
assert.Equal(t, true, um[1].Blocked.Status)
assert.Equal(t, "test@example.org", um[1].Details.Email)
assert.Equal(t, "user3", um[2].ID)
assert.Equal(t, "test@example.org", um[2].Details.Email)
assert.Equal(t, 1, len(pm))
assert.Equal(t, "https://radio-t.com", pm[0].URL)
@@ -743,7 +757,7 @@ func TestService_SetMetas(t *testing.T) {
err := b.SetMetas("radio-t", umetas, pmetas)
assert.NoError(t, err, "empty metas")
um1 := UserMetaData{ID: "user1", Verified: true}
um1 := UserMetaData{ID: "user1", Verified: true, Details: engine.UserDetailEntry{Email: "test@example.org"}}
um2 := UserMetaData{ID: "user2"}
um2.Blocked.Status = true
um2.Blocked.Until = time.Now().AddDate(0, 1, 1)
@@ -755,6 +769,9 @@ func TestService_SetMetas(t *testing.T) {
assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}))
assert.True(t, b.IsVerified("radio-t", "user1"))
assert.True(t, b.IsBlocked("radio-t", "user2"))
val, err := b.Engine.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Detail: engine.UserEmail})
assert.NoError(t, err)
assert.Equal(t, []engine.UserDetailEntry{{UserID: "user1", Email: "test@example.org"}}, val)
}
func TestService_IsAdmin(t *testing.T) {