diff --git a/app/main.go b/app/main.go index 3f3aced9..e46b20a5 100644 --- a/app/main.go +++ b/app/main.go @@ -13,13 +13,14 @@ import ( "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" "github.com/pkg/errors" + "github.com/umputun/remark/app/store/engine" + "github.com/umputun/remark/app/store/service" "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest" "github.com/umputun/remark/app/rest/api" "github.com/umputun/remark/app/rest/auth" "github.com/umputun/remark/app/rest/avatar" - "github.com/umputun/remark/app/store" ) var opts struct { @@ -74,7 +75,7 @@ func main() { log.Printf("[WARN] running in dev mode") } - dataService := store.Service{ + dataService := service.DataStore{ Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.SecretKey, @@ -142,12 +143,12 @@ func activateBackup(exporter migrator.Exporter) { } // makeBoltStore creates store for all sites -func makeBoltStore(siteNames []string) store.Interface { - sites := []store.BoltSite{} +func makeBoltStore(siteNames []string) engine.Interface { + sites := []engine.BoltSite{} for _, site := range siteNames { - sites = append(sites, store.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", opts.BoltPath, site)}) + sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", opts.BoltPath, site)}) } - result, err := store.NewBoltDB(bolt.Options{Timeout: 30 * time.Second}, sites...) + result, err := engine.NewBoltDB(bolt.Options{Timeout: 30 * time.Second}, sites...) if err != nil { log.Fatalf("[ERROR] can't initialize data store, %+v", err) } diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go index 74295f30..7eeafd83 100644 --- a/app/migrator/disqus.go +++ b/app/migrator/disqus.go @@ -10,11 +10,12 @@ import ( "github.com/pkg/errors" "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/service" ) // Disqus implements Importer from disqus xml type Disqus struct { - DataStore store.Interface + DataStore *service.DataStore } type disqusThread struct { diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go index bce46a57..d547bbec 100644 --- a/app/migrator/disqus_test.go +++ b/app/migrator/disqus_test.go @@ -8,16 +8,19 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/require" + "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/engine" + "github.com/umputun/remark/app/store/service" "github.com/stretchr/testify/assert" - "github.com/umputun/remark/app/store" ) func TestDisqus_Import(t *testing.T) { defer os.Remove("/tmp/remark-test.db") - dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - d := Disqus{DataStore: dataStore} + dataStore := service.DataStore{Interface: b} + d := Disqus{DataStore: &dataStore} size, err := d.Import(strings.NewReader(xmlTest), "test") assert.Nil(t, err) assert.Equal(t, 3, size) @@ -33,7 +36,7 @@ func TestDisqus_Import(t *testing.T) { assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator) assert.Equal(t, "Dmitry Noname", c.User.Name) assert.Equal(t, "disqus_8799342cdf328253e03313958ffc6a433659d7ff", c.User.ID) - assert.Equal(t, "89.89.89.139", c.User.IP) + assert.Equal(t, "96243f024cf6ad42b66f0c72709ae20b5d10ec14", c.User.IP) posts, err := dataStore.List("test", 0, 0) assert.Nil(t, err) diff --git a/app/migrator/migrator.go b/app/migrator/migrator.go index ef6e6514..4ed884d7 100644 --- a/app/migrator/migrator.go +++ b/app/migrator/migrator.go @@ -10,7 +10,7 @@ import ( "github.com/pkg/errors" - "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/service" ) // Importer defines interface to convert posts from external sources @@ -25,7 +25,7 @@ type Exporter interface { // ImportParams defines everything needed to run import type ImportParams struct { - DataStore *store.Service + DataStore *service.DataStore InputFile string Provider string SiteID string diff --git a/app/migrator/migrator_test.go b/app/migrator/migrator_test.go index bc849fba..b4559eee 100644 --- a/app/migrator/migrator_test.go +++ b/app/migrator/migrator_test.go @@ -8,8 +8,9 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/app/store/service" - "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/engine" ) func TestMigrator_ImportDisqus(t *testing.T) { @@ -21,9 +22,9 @@ func TestMigrator_ImportDisqus(t *testing.T) { err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTest), 0600) require.Nil(t, err) - b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &store.Service{Interface: b} + dataStore := &service.DataStore{Interface: b} size, err := ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/disqus-test.xml", @@ -50,9 +51,9 @@ func TestMigrator_ImportRemark(t *testing.T) { err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600) require.Nil(t, err) - b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) require.Nil(t, err, "create store") - dataStore := &store.Service{Interface: b} + dataStore := &service.DataStore{Interface: b} size, err := ImportComments(ImportParams{ DataStore: dataStore, diff --git a/app/migrator/remark.go b/app/migrator/remark.go index ed7a1a48..39937fc6 100644 --- a/app/migrator/remark.go +++ b/app/migrator/remark.go @@ -8,13 +8,14 @@ import ( "log" "github.com/pkg/errors" + "github.com/umputun/remark/app/store/service" "github.com/umputun/remark/app/store" ) // Remark implements exporter and importer for internal store format type Remark struct { - DataStore *store.Service + DataStore *service.DataStore } // Export all comments to writer as json strings. Each comment is one string, separated by "\n" diff --git a/app/migrator/remark_test.go b/app/migrator/remark_test.go index ff20f224..2c9a02d0 100644 --- a/app/migrator/remark_test.go +++ b/app/migrator/remark_test.go @@ -9,8 +9,9 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" - "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/engine" + "github.com/umputun/remark/app/store/service" ) var testDb = "/tmp/test-remark.db" @@ -41,9 +42,9 @@ func TestRemark_Import(t *testing.T) { buf.WriteString(r2) os.Remove(testDb) - b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{SiteID: "radio-t", FileName: testDb}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - r := Remark{DataStore: &store.Service{Interface: b}} + r := Remark{DataStore: &service.DataStore{Interface: b}} size, err := r.Import(buf, "radio-t") assert.Nil(t, err) assert.Equal(t, 2, size) @@ -57,13 +58,13 @@ func TestRemark_Import(t *testing.T) { } // makes new boltdb, put two records -func prep(t *testing.T) *store.Service { +func prep(t *testing.T) *service.DataStore { os.Remove(testDb) - boltStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{SiteID: "radio-t", FileName: testDb}) + boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - b := &store.Service{Interface: boltStore} + b := &service.DataStore{Interface: boltStore} comment := store.Comment{ ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc", diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 4f4c93f6..1f55943d 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -14,11 +14,12 @@ import ( "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest" "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/service" ) // admin provides router for all requests available for admin users only type admin struct { - dataService store.Service + dataService service.DataStore exporter migrator.Exporter cache rest.LoadingCache defAvatarURL string diff --git a/app/rest/api/import_test.go b/app/rest/api/import_test.go index 2f951652..9050a5dc 100644 --- a/app/rest/api/import_test.go +++ b/app/rest/api/import_test.go @@ -14,9 +14,10 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/app/store/engine" + "github.com/umputun/remark/app/store/service" "github.com/umputun/remark/app/migrator" - "github.com/umputun/remark/app/store" ) func TestImport(t *testing.T) { @@ -52,9 +53,9 @@ func TestImportRejected(t *testing.T) { } func prepImportSrv(t *testing.T) (srv *Import, port int) { - b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: testDb, SiteID: "radio-t"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) - dataStore := &store.Service{Interface: b} + dataStore := &service.DataStore{Interface: b} srv = &Import{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, NativeImporter: &migrator.Remark{DataStore: dataStore}, diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 5dcaee93..50477b1d 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -19,6 +19,7 @@ import ( "github.com/go-chi/render" "github.com/gorilla/context" "github.com/pkg/errors" + "github.com/umputun/remark/app/store/service" "gopkg.in/russross/blackfriday.v2" "github.com/umputun/remark/app/migrator" @@ -30,7 +31,7 @@ import ( // Rest is a rest access server type Rest struct { Version string - DataService store.Service + DataService service.DataStore Authenticator auth.Authenticator Exporter migrator.Exporter Cache rest.LoadingCache @@ -238,7 +239,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - editReq := store.EditRequest{ + editReq := service.EditRequest{ Text: string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))), // render markdown Orig: edit.Text, Summary: edit.Summary, diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go index ac941d29..7c1de080 100644 --- a/app/rest/api/rest_test.go +++ b/app/rest/api/rest_test.go @@ -17,11 +17,13 @@ import ( "github.com/gorilla/sessions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/app/rest/avatar" + "github.com/umputun/remark/app/store/service" "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest/auth" + "github.com/umputun/remark/app/rest/avatar" "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/engine" ) var testDb = "/tmp/test-remark.db" @@ -147,7 +149,7 @@ func TestServer_CreateAndGet(t *testing.T) { assert.Equal(t, `

test 123 http://radio-t.com

`+"\n", comment.Text) assert.Equal(t, "**test** *123* http://radio-t.com", comment.Orig) assert.Equal(t, store.User{Name: "developer one", ID: "dev", - Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "ea64bfc178468d943ca5b836e2e700c335404973"}, + Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"}, comment.User) t.Logf("%+v", comment) } @@ -462,11 +464,11 @@ func TestServer_FileServer(t *testing.T) { } func prep(t *testing.T) (srv *Rest, port int) { - b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: testDb, SiteID: "radio-t"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) - dataStore := &store.Service{Interface: b} + dataStore := service.DataStore{Interface: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, Secret: "123456"} srv = &Rest{ - DataService: store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, MaxCommentSize: 4000}, + DataService: dataStore, Authenticator: auth.Authenticator{ SessionStore: sessions.NewFilesystemStore("/tmp", []byte("blah")), DevPasswd: "password", @@ -474,14 +476,14 @@ func prep(t *testing.T) (srv *Rest, port int) { AvatarProxy: &avatar.Proxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"}, Admins: []string{"a1", "a2"}, }, - Exporter: &migrator.Remark{DataStore: dataStore}, + Exporter: &migrator.Remark{DataStore: &dataStore}, Cache: &mockCache{}, WebRoot: "/tmp", } importSrv := &Import{ - DisqusImporter: &migrator.Disqus{DataStore: dataStore}, - NativeImporter: &migrator.Remark{DataStore: dataStore}, + DisqusImporter: &migrator.Disqus{DataStore: &dataStore}, + NativeImporter: &migrator.Remark{DataStore: &dataStore}, Cache: &mockCache{}, } diff --git a/app/rest/auth/provider.go b/app/rest/auth/provider.go index 5e2a997d..013e5b7d 100644 --- a/app/rest/auth/provider.go +++ b/app/rest/auth/provider.go @@ -15,10 +15,10 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" "github.com/gorilla/sessions" - "github.com/umputun/remark/app/rest/avatar" "golang.org/x/oauth2" "github.com/umputun/remark/app/rest" + "github.com/umputun/remark/app/rest/avatar" "github.com/umputun/remark/app/store" ) diff --git a/app/store/comment.go b/app/store/comment.go index 21ede191..0907331f 100644 --- a/app/store/comment.go +++ b/app/store/comment.go @@ -1,14 +1,8 @@ package store import ( - "crypto/hmac" - "crypto/sha1" - "fmt" - "hash/crc64" "html/template" - "log" "regexp" - "strconv" "time" "github.com/microcosm-cc/bluemonday" @@ -36,16 +30,6 @@ type Locator struct { URL string `json:"url"` } -// User holds user-related info -type User struct { - Name string `json:"name"` - ID string `json:"id"` - Picture string `json:"picture"` - Admin bool `json:"admin"` - Blocked bool `json:"block,omitempty"` - IP string `json:"ip,omitempty"` -} - // Edit indication type Edit struct { Timestamp time.Time `json:"time"` @@ -97,33 +81,3 @@ func (c *Comment) Sanitize() { c.User.Name = template.HTMLEscapeString(c.User.Name) c.User.Picture = p.Sanitize(c.User.Picture) } - -// hashIP replace IP field with hashed hmac -func (u *User) hashIP(secret string) { - - hashVal := func(val string) string { - if _, err := strconv.ParseUint(val, 16, 64); err == nil || val == "" { - return val // already hashed - } - key := []byte(secret) - h := hmac.New(sha1.New, key) - if _, err := h.Write([]byte(val)); err != nil { - log.Printf("[WARN] can't hash ip, %s", err) - } - return fmt.Sprintf("%x", h.Sum(nil)) - } - - u.IP = hashVal(u.IP) -} - -// EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases -// we need hashing for parts of id, in some others hashing for non-User values. -func EncodeID(id string) string { - h := sha1.New() - if _, err := h.Write([]byte(id)); err != nil { - // fail back to crc64 - log.Printf("[WARN] can't hash id %s, %s", id, err) - return fmt.Sprintf("%x", crc64.Checksum([]byte(id), crc64.MakeTable(crc64.ECMA))) - } - return fmt.Sprintf("%x", h.Sum(nil)) -} diff --git a/app/store/comment_test.go b/app/store/comment_test.go index e37989e6..33713083 100644 --- a/app/store/comment_test.go +++ b/app/store/comment_test.go @@ -58,18 +58,3 @@ func TestComment_PrepareUntrusted(t *testing.T) { assert.Equal(t, User{ID: "username"}, comment.User) } - -func TestComment_EncodeID(t *testing.T) { - tbl := []struct { - id string - hash string - }{ - {"myid", "6e34471f84557e1713012d64a7477c71bfdac631"}, - {"", "da39a3ee5e6b4b0d3255bfef95601890afd80709"}, - {"blah blah", "135a1e01bae742c4a576b20fd41a683f6483ca43"}, - } - - for i, tt := range tbl { - assert.Equal(t, tt.hash, EncodeID(tt.id), "case #%d", i) - } -} diff --git a/app/store/bolt.go b/app/store/engine/bolt.go similarity index 91% rename from app/store/bolt.go rename to app/store/engine/bolt.go index 65062c8b..1b14b199 100644 --- a/app/store/bolt.go +++ b/app/store/engine/bolt.go @@ -1,4 +1,4 @@ -package store +package engine import ( "encoding/json" @@ -10,6 +10,8 @@ import ( "github.com/coreos/bbolt" "github.com/pkg/errors" + + "github.com/umputun/remark/app/store" ) // BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe. @@ -78,7 +80,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) { } // Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket -func (b *BoltDB) Create(comment Comment) (commentID string, err error) { +func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { bdb, err := b.db(comment.Locator.SiteID) if err != nil { @@ -135,7 +137,7 @@ func (b *BoltDB) Create(comment Comment) (commentID string, err error) { // Delete removes comment, by locator from the store. // Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. // From last bucket removed for real. -func (b *BoltDB) Delete(locator Locator, commentID string) error { +func (b *BoltDB) Delete(locator store.Locator, commentID string) error { bdb, err := b.db(locator.SiteID) if err != nil { @@ -204,8 +206,8 @@ func (b *BoltDB) DeleteAll(siteID string) error { } // Find returns all comments for post and sorts results -func (b *BoltDB) Find(locator Locator, sortFld string) (comments []Comment, err error) { - comments = []Comment{} +func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) { + comments = []store.Comment{} bdb, err := b.db(locator.SiteID) if err != nil { @@ -220,7 +222,7 @@ func (b *BoltDB) Find(locator Locator, sortFld string) (comments []Comment, err } return bucket.ForEach(func(k, v []byte) error { - comment := Comment{} + comment := store.Comment{} if e := json.Unmarshal(v, &comment); e != nil { return errors.Wrap(e, "failed to unmarshal") } @@ -234,7 +236,7 @@ func (b *BoltDB) Find(locator Locator, sortFld string) (comments []Comment, err } // Last returns up to max last comments for given siteID -func (b *BoltDB) Last(siteID string, max int) (comments []Comment, err error) { +func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) { if max > lastLimit || max == 0 { max = lastLimit @@ -278,7 +280,7 @@ func (b *BoltDB) Last(siteID string, max int) (comments []Comment, err error) { } // Count returns number of comments for locator -func (b *BoltDB) Count(locator Locator) (count int, err error) { +func (b *BoltDB) Count(locator store.Locator) (count int, err error) { bdb, err := b.db(locator.SiteID) if err != nil { @@ -336,8 +338,8 @@ func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { // Blocked get lists of blocked users for given site // bucket uses userID: -func (b *BoltDB) Blocked(siteID string) (users []BlockedUser, err error) { - users = []BlockedUser{} +func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { + users = []store.BlockedUser{} bdb, err := b.db(siteID) if err != nil { return nil, err @@ -357,7 +359,7 @@ func (b *BoltDB) Blocked(siteID string) (users []BlockedUser, err error) { userName = userComments[0].User.Name } - users = append(users, BlockedUser{ID: string(k), Name: userName, Timestamp: ts}) + users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts}) return nil }) }) @@ -367,7 +369,7 @@ func (b *BoltDB) Blocked(siteID string) (users []BlockedUser, err error) { // List returns list of all commented posts with counters // uses count bucket to get number of comments -func (b BoltDB) List(siteID string, limit, skip int) (list []PostInfo, err error) { +func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) { bdb, err := b.db(siteID) if err != nil { @@ -389,7 +391,7 @@ func (b BoltDB) List(siteID string, limit, skip int) (list []PostInfo, err error if e != nil { return e } - list = append(list, PostInfo{URL: postURL, Count: count}) + list = append(list, store.PostInfo{URL: postURL, Count: count}) if limit > 0 && len(list) >= limit { break } @@ -402,9 +404,9 @@ func (b BoltDB) List(siteID string, limit, skip int) (list []PostInfo, err error // User extracts all comments for given site and given userID // "users" bucket has sub-bucket for each userID, and keeps it as ts:ref -func (b *BoltDB) User(siteID string, userID string, limit int) (comments []Comment, totalComments int, err error) { +func (b *BoltDB) User(siteID string, userID string, limit int) (comments []store.Comment, totalComments int, err error) { - comments = []Comment{} + comments = []store.Comment{} commentRefs := []string{} bdb, err := b.db(siteID) @@ -445,7 +447,7 @@ func (b *BoltDB) User(siteID string, userID string, limit int) (comments []Comme if e != nil { return comments, totalComments, errors.Wrapf(e, "can't parse reference %s", v) } - if c, e := b.Get(Locator{SiteID: siteID, URL: url}, commentID); e == nil { + if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil { comments = append(comments, c) } } @@ -454,7 +456,7 @@ func (b *BoltDB) User(siteID string, userID string, limit int) (comments []Comme } // Get returns comment for locator.URL and commentID string -func (b *BoltDB) Get(locator Locator, commentID string) (comment Comment, err error) { +func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { bdb, err := b.db(locator.SiteID) if err != nil { @@ -473,7 +475,7 @@ func (b *BoltDB) Get(locator Locator, commentID string) (comment Comment, err er } // Put updates comment for locator.URL with mutable part of comment -func (b *BoltDB) Put(locator Locator, comment Comment) error { +func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error { if curComment, err := b.Get(locator, comment.ID); err == nil { // preserve immutable fields @@ -533,7 +535,7 @@ func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) } // save comment to key for bucket. Should run in update tx -func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, comment Comment) (err error) { +func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, comment store.Comment) (err error) { jdata, jerr := json.Marshal(&comment) if jerr != nil { return errors.Wrap(jerr, "can't marshal comment") @@ -545,7 +547,7 @@ func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, comment Comment) (err error) } // load comment by key from bucket. Should run in view tx -func (b *BoltDB) load(bkt *bolt.Bucket, key []byte) (comment Comment, err error) { +func (b *BoltDB) load(bkt *bolt.Bucket, key []byte) (comment store.Comment, err error) { commentVal := bkt.Get(key) if commentVal == nil { return comment, errors.Errorf("no comment for %s", key) @@ -590,7 +592,7 @@ func (b *BoltDB) db(siteID string) (*bolt.DB, error) { } // makeRef creates reference combining url and comment id -func (b *BoltDB) makeRef(comment Comment) []byte { +func (b *BoltDB) makeRef(comment store.Comment) []byte { return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID)) } diff --git a/app/store/bolt_test.go b/app/store/engine/bolt_test.go similarity index 75% rename from app/store/bolt_test.go rename to app/store/engine/bolt_test.go index f4ab80da..5a8edc9a 100644 --- a/app/store/bolt_test.go +++ b/app/store/engine/bolt_test.go @@ -1,4 +1,4 @@ -package store +package engine import ( "os" @@ -7,6 +7,8 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" + + "github.com/umputun/remark/app/store" ) var testDb = "/tmp/test-remark.db" @@ -15,14 +17,14 @@ func TestBoltDB_CreateAndFind(t *testing.T) { var b = prep(t) defer os.Remove(testDb) - res, err := b.Find(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") + res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") assert.Nil(t, err) assert.Equal(t, 2, len(res)) assert.Equal(t, `some text, link`, res[0].Text) assert.Equal(t, "user1", res[0].User.ID) t.Log(res[0].ID) - _, err = b.Create(Comment{ID: res[0].ID, Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) + _, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) assert.NotNil(t, err) assert.Equal(t, "key id-1 already in store", err.Error()) } @@ -31,7 +33,7 @@ func TestBoltDB_Delete(t *testing.T) { defer os.Remove(testDb) b := prep(t) - loc := Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} res, err := b.Find(loc, "time") assert.Nil(t, err) assert.Equal(t, 2, len(res), "initially 2 comments") @@ -56,7 +58,7 @@ func TestBoltDB_DeleteAll(t *testing.T) { defer os.Remove(testDb) b := prep(t) - loc := Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} res, err := b.Find(loc, "time") assert.Nil(t, err) assert.Equal(t, 2, len(res), "initially 2 comments") @@ -68,7 +70,7 @@ func TestBoltDB_DeleteAll(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 0, len(comments), "nothing left") - c, err := b.Count(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) + c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) assert.Nil(t, err) assert.Equal(t, 0, c, "0 count") } @@ -77,22 +79,22 @@ func TestBoltDB_Get(t *testing.T) { defer os.Remove(testDb) b := prep(t) - res, err := b.Find(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") + res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") assert.Nil(t, err) assert.Equal(t, 2, len(res)) - comment, err := b.Get(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) + comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) assert.Nil(t, err) assert.Equal(t, "some text2", comment.Text) - comment, err = b.Get(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") + comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") assert.NotNil(t, err) } func TestBoltDB_Put(t *testing.T) { defer os.Remove(testDb) b := prep(t) - loc := Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} res, err := b.Find(loc, "time") assert.Nil(t, err) assert.Equal(t, 2, len(res)) @@ -129,7 +131,7 @@ func TestBoltDB_Count(t *testing.T) { defer os.Remove(testDb) b := prep(t) - c, err := b.Count(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) + c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) assert.Nil(t, err) assert.Equal(t, 2, c) } @@ -171,27 +173,27 @@ func TestBoltDB_List(t *testing.T) { b := prep(t) // two comments for https://radio-t.com // add one more for https://radio-t.com/2 - comment := Comment{ + comment := store.Comment{ ID: "12345", Text: `some text, link`, Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: User{ID: "user1", Name: "user name"}, + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, } _, err := b.Create(comment) assert.Nil(t, err) res, err := b.List("radio-t", 0, 0) assert.Nil(t, err) - assert.Equal(t, []PostInfo{{URL: "https://radio-t.com/2", Count: 1}, {URL: "https://radio-t.com", Count: 2}}, res) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1}, {URL: "https://radio-t.com", Count: 2}}, res) res, err = b.List("radio-t", 1, 0) assert.Nil(t, err) - assert.Equal(t, []PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res) res, err = b.List("radio-t", 1, 1) assert.Nil(t, err) - assert.Equal(t, []PostInfo{{URL: "https://radio-t.com", Count: 2}}, res) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2}}, res) } func TestBoltDB_GetForUser(t *testing.T) { @@ -220,22 +222,22 @@ func prep(t *testing.T) *BoltDB { assert.Nil(t, err) b := boltStore - comment := Comment{ + comment := store.Comment{ ID: "id-1", Text: `some text, link`, Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: User{ID: "user1", Name: "user name"}, + Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, } _, err = b.Create(comment) assert.Nil(t, err) - comment = Comment{ + comment = store.Comment{ ID: "id-2", Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: User{ID: "user1", Name: "user name"}, + Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, } _, err = b.Create(comment) assert.Nil(t, err) diff --git a/app/store/store.go b/app/store/engine/engine.go similarity index 53% rename from app/store/store.go rename to app/store/engine/engine.go index 83d2159c..b2714957 100644 --- a/app/store/store.go +++ b/app/store/engine/engine.go @@ -1,12 +1,16 @@ -package store - -//go:generate sh -c "mockery -inpkg -name Interface -print > file.tmp && mv file.tmp store_mock.go" +// Package engine defines interfaces each supported storage should implement. +// Includes default implementation with boltdb +package engine import ( "sort" "strings" + + R "github.com/umputun/remark/app/store" ) +//go:generate sh -c "mockery -inpkg -name Interface -print > file.tmp && mv file.tmp engine_mock.go" + // Interface combines all store interfaces type Interface interface { Accessor @@ -15,26 +19,26 @@ type Interface interface { // Accessor defines all usual access ops avail for regular user type Accessor interface { - Create(comment Comment) (commentID string, err error) // create new comment, avoid dups by id - Get(locator Locator, commentID string) (comment Comment, err error) // get comment by id - Put(locator Locator, comment Comment) error // update comment, mutable parts only - Find(locator Locator, sort string) ([]Comment, error) // find comments for locator - Last(siteID string, limit int) ([]Comment, error) // last comments for given site, sorted by time - User(siteID string, userID string, limit int) ([]Comment, int, error) // comments by user, sorted by time - Count(locator Locator) (int, error) // number of comments for the post - List(siteID string, limit int, skip int) ([]PostInfo, error) // list of commented posts + Create(comment R.Comment) (commentID string, err error) // create new comment, avoid dups by id + Get(locator R.Locator, commentID string) (comment R.Comment, err error) // get comment by id + Put(locator R.Locator, comment R.Comment) error // update comment, mutable parts only + Find(locator R.Locator, sort string) ([]R.Comment, error) // find comments for locator + Last(siteID string, limit int) ([]R.Comment, error) // last comments for given site, sorted by time + User(siteID string, userID string, limit int) ([]R.Comment, int, error) // comments by user, sorted by time + Count(locator R.Locator) (int, error) // number of comments for the post + List(siteID string, limit int, skip int) ([]R.PostInfo, error) // list of commented posts } // Admin defines all store ops avail for admin only type Admin interface { - Delete(locator Locator, commentID string) error // delete comment by id + Delete(locator R.Locator, commentID string) error // delete comment by id DeleteAll(siteID string) error // delete all data from site SetBlock(siteID string, userID string, status bool) error // block or unblock user IsBlocked(siteID string, userID string) bool // check if user blocked - Blocked(siteID string) ([]BlockedUser, error) // get list of blocked users + Blocked(siteID string) ([]R.BlockedUser, error) // get list of blocked users } -func sortComments(comments []Comment, sortFld string) []Comment { +func sortComments(comments []R.Comment, sortFld string) []R.Comment { sort.Slice(comments, func(i, j int) bool { switch sortFld { case "+time", "-time", "time", "+active", "-active", "active": diff --git a/app/store/store_test.go b/app/store/engine/engine_test.go similarity index 86% rename from app/store/store_test.go rename to app/store/engine/engine_test.go index c3df0058..6b73f3ed 100644 --- a/app/store/store_test.go +++ b/app/store/engine/engine_test.go @@ -1,14 +1,16 @@ -package store +package engine import ( "testing" "time" "github.com/stretchr/testify/assert" + + "github.com/umputun/remark/app/store" ) -func TestStore_sortComments(t *testing.T) { - cc := []Comment{ +func TestEngine_sortComments(t *testing.T) { + cc := []store.Comment{ {ID: "1", Score: 5, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)}, {ID: "2", Score: 4, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)}, {ID: "3", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)}, diff --git a/app/store/service.go b/app/store/service/service.go similarity index 75% rename from app/store/service.go rename to app/store/service/service.go index 28583521..97e862e0 100644 --- a/app/store/service.go +++ b/app/store/service/service.go @@ -1,15 +1,18 @@ -package store +package service import ( "time" "github.com/google/uuid" "github.com/pkg/errors" + + R "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/engine" ) -// Service wraps store.Interface with additional methods -type Service struct { - Interface +// DataStore wraps store.Interface with additional methods +type DataStore struct { + engine.Interface EditDuration time.Duration Secret string MaxCommentSize int @@ -18,7 +21,7 @@ type Service struct { const defaultCommentMaxSize = 2000 // Create prepares comment and forward to Interface.Create -func (s *Service) Create(comment Comment) (commentID string, err error) { +func (s *DataStore) Create(comment R.Comment) (commentID string, err error) { // fill ID and time if empty if comment.ID == "" { comment.ID = uuid.New().String() @@ -32,13 +35,13 @@ func (s *Service) Create(comment Comment) (commentID string, err error) { } comment.Sanitize() // clear potentially dangerous js from all parts of comment - comment.User.hashIP(s.Secret) // replace ip by hash + comment.User.HashIP(s.Secret) // replace ip by hash return s.Interface.Create(comment) } // SetPin pin/un-pin comment as special -func (s *Service) SetPin(locator Locator, commentID string, status bool) error { +func (s *DataStore) SetPin(locator R.Locator, commentID string, status bool) error { comment, err := s.Get(locator, commentID) if err != nil { return err @@ -48,7 +51,7 @@ func (s *Service) SetPin(locator Locator, commentID string, status bool) error { } // Vote for comment by id and locator -func (s *Service) Vote(locator Locator, commentID string, userID string, val bool) (comment Comment, err error) { +func (s *DataStore) Vote(locator R.Locator, commentID string, userID string, val bool) (comment R.Comment, err error) { comment, err = s.Get(locator, commentID) if err != nil { @@ -96,7 +99,7 @@ type EditRequest struct { } // EditComment to edit text and update Edit info -func (s *Service) EditComment(locator Locator, commentID string, req EditRequest) (comment Comment, err error) { +func (s *DataStore) EditComment(locator R.Locator, commentID string, req EditRequest) (comment R.Comment, err error) { comment, err = s.Get(locator, commentID) if err != nil { return comment, err @@ -113,7 +116,7 @@ func (s *Service) EditComment(locator Locator, commentID string, req EditRequest comment.Text = req.Text comment.Orig = req.Orig - comment.Edit = &Edit{ + comment.Edit = &R.Edit{ Timestamp: time.Now(), Summary: req.Summary, } @@ -124,18 +127,18 @@ func (s *Service) EditComment(locator Locator, commentID string, req EditRequest } // Counts returns postID+count list for given comments -func (s *Service) Counts(siteID string, postIDs []string) ([]PostInfo, error) { - res := []PostInfo{} +func (s *DataStore) Counts(siteID string, postIDs []string) ([]R.PostInfo, error) { + res := []R.PostInfo{} for _, p := range postIDs { - if c, err := s.Count(Locator{SiteID: siteID, URL: p}); err == nil { - res = append(res, PostInfo{URL: p, Count: c}) + if c, err := s.Count(R.Locator{SiteID: siteID, URL: p}); err == nil { + res = append(res, R.PostInfo{URL: p, Count: c}) } } return res, nil } // ValidateComment checks if comment size below max and user fields set -func (s *Service) ValidateComment(c *Comment) error { +func (s *DataStore) ValidateComment(c *R.Comment) error { maxSize := s.MaxCommentSize if s.MaxCommentSize <= 0 { maxSize = defaultCommentMaxSize diff --git a/app/store/service_test.go b/app/store/service/service_test.go similarity index 53% rename from app/store/service_test.go rename to app/store/service/service_test.go index ee62f67f..bf0f2fad 100644 --- a/app/store/service_test.go +++ b/app/store/service/service_test.go @@ -1,4 +1,4 @@ -package store +package service import ( "fmt" @@ -10,21 +10,26 @@ import ( "github.com/coreos/bbolt" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + + R "github.com/umputun/remark/app/store" + "github.com/umputun/remark/app/store/engine" ) +var testDb = "/tmp/test-remark.db" + func TestService_CreateFromEmpty(t *testing.T) { defer os.Remove(testDb) - b := Service{Interface: prep(t), Secret: "secret 123"} - comment := Comment{ + b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"} + comment := R.Comment{ Text: "text", - User: User{IP: "192.168.1.1", ID: "user", Name: "name"}, - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: R.User{IP: "192.168.1.1", ID: "user", Name: "name"}, + Locator: R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, } id, err := b.Create(comment) assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Get(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) + res, err := b.Get(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "text", res.Text) @@ -37,19 +42,19 @@ func TestService_CreateFromEmpty(t *testing.T) { func TestService_CreateFromPartial(t *testing.T) { defer os.Remove(testDb) - b := Service{Interface: prep(t), Secret: "secret 123"} - comment := Comment{ + b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"} + comment := R.Comment{ Text: "text", Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC), Votes: map[string]bool{"u1": true, "u2": false}, - User: User{IP: "192.168.1.1", ID: "user", Name: "name"}, - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: R.User{IP: "192.168.1.1", ID: "user", Name: "name"}, + Locator: R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, } id, err := b.Create(comment) assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Get(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) + res, err := b.Get(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "text", res.Text) @@ -62,12 +67,12 @@ func TestService_CreateFromPartial(t *testing.T) { func TestService_Vote(t *testing.T) { defer os.Remove(testDb) - b := Service{Interface: prep(t)} + b := DataStore{Interface: prepStoreEngine(t)} - comment := Comment{ + comment := R.Comment{ Text: "text", - User: User{IP: "192.168.1.1", ID: "user", Name: "name"}, - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: R.User{IP: "192.168.1.1", ID: "user", Name: "name"}, + Locator: R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, } _, err := b.Create(comment) assert.NoError(t, err) @@ -79,15 +84,15 @@ func TestService_Vote(t *testing.T) { assert.Equal(t, 0, res[0].Score) assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially") - c, err := b.Vote(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true) + c, err := b.Vote(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true) assert.Nil(t, err) assert.Equal(t, 1, c.Score) assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +") - c, err = b.Vote(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true) + c, err = b.Vote(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true) assert.NotNil(t, err, "self-voting not allowed") - _, err = b.Vote(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true) + _, err = b.Vote(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true) assert.NotNil(t, err, "double-voting rejected") assert.True(t, strings.HasPrefix(err.Error(), "user user1 already voted")) @@ -96,7 +101,7 @@ func TestService_Vote(t *testing.T) { assert.Equal(t, 3, len(res)) assert.Equal(t, 1, res[0].Score) - _, err = b.Vote(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false) + _, err = b.Vote(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false) assert.Nil(t, err, "vote reset") res, err = b.Last("radio-t", 0) assert.Nil(t, err) @@ -107,7 +112,7 @@ func TestService_Vote(t *testing.T) { func TestService_Pin(t *testing.T) { defer os.Remove(testDb) - b := Service{Interface: prep(t)} + b := DataStore{Interface: prepStoreEngine(t)} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -115,23 +120,23 @@ func TestService_Pin(t *testing.T) { assert.Equal(t, 2, len(res)) assert.Equal(t, false, res[0].Pin) - err = b.SetPin(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true) + err = b.SetPin(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true) assert.Nil(t, err) - c, err := b.Get(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + c, err := b.Get(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) assert.Nil(t, err) assert.Equal(t, true, c.Pin) - err = b.SetPin(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false) + err = b.SetPin(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false) assert.Nil(t, err) - c, err = b.Get(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + c, err = b.Get(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) assert.Nil(t, err) assert.Equal(t, false, c.Pin) } func TestService_EditComment(t *testing.T) { defer os.Remove(testDb) - b := Service{Interface: prep(t)} + b := DataStore{Interface: prepStoreEngine(t)} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -139,66 +144,53 @@ func TestService_EditComment(t *testing.T) { assert.Equal(t, 2, len(res)) assert.Nil(t, res[0].Edit) - comment, err := b.EditComment(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, + comment, err := b.EditComment(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"}) assert.Nil(t, err) assert.Equal(t, "my edit", comment.Edit.Summary) assert.Equal(t, "xxx", comment.Text) assert.Equal(t, "yyy", comment.Orig) - c, err := b.Get(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + c, err := b.Get(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) assert.Nil(t, err) assert.Equal(t, "my edit", c.Edit.Summary) assert.Equal(t, "xxx", c.Text) - _, err = b.EditComment(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, + _, err = b.EditComment(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"}) assert.NotNil(t, err, "allow edit once") } func TestService_EditCommentDurationFailed(t *testing.T) { defer os.Remove(testDb) - - blt, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"}) - assert.Nil(t, err) - - comment := Comment{ - ID: "id-1", - Text: `some text, link`, - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: User{ID: "user1", Name: "user name"}, - } - _, err = blt.Create(comment) - assert.Nil(t, err) - - b := Service{Interface: blt, EditDuration: 100 * time.Millisecond} + b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) assert.Nil(t, err) - assert.Equal(t, 1, len(res)) + assert.Equal(t, 2, len(res)) assert.Nil(t, res[0].Edit) time.Sleep(time.Second) - _, err = b.EditComment(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, + _, err = b.EditComment(R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"}) assert.NotNil(t, err) } func TestService_ValidateComment(t *testing.T) { - b := Service{MaxCommentSize: 2000} + b := DataStore{MaxCommentSize: 2000} longText := fmt.Sprintf("%4000s", "X") tbl := []struct { - inp Comment + inp R.Comment err error }{ - {inp: Comment{}, err: errors.New("empty comment text")}, - {inp: Comment{Orig: "something blah", User: User{ID: "myid", Name: "name"}}, err: nil}, - {inp: Comment{Orig: "something blah", User: User{ID: "myid"}}, err: errors.New("empty user info")}, - {inp: Comment{Orig: longText, User: User{ID: "myid", Name: "name"}}, err: errors.New("comment text exceeded max allowed size 2000 (4000)")}, + {inp: R.Comment{}, err: errors.New("empty comment text")}, + {inp: R.Comment{Orig: "something blah", User: R.User{ID: "myid", Name: "name"}}, err: nil}, + {inp: R.Comment{Orig: "something blah", User: R.User{ID: "myid"}}, err: errors.New("empty user info")}, + {inp: R.Comment{Orig: longText, User: R.User{ID: "myid", Name: "name"}}, err: errors.New("comment text exceeded max allowed size 2000 (4000)")}, } for n, tt := range tbl { @@ -213,29 +205,60 @@ func TestService_ValidateComment(t *testing.T) { func TestService_Counts(t *testing.T) { defer os.Remove(testDb) - b := prep(t) // two comments for https://radio-t.com + b := prepStoreEngine(t) // two comments for https://radio-t.com // add one more for https://radio-t.com/2 - comment := Comment{ + comment := R.Comment{ ID: "123456", Text: `some text, link`, Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: User{ID: "user1", Name: "user name"}, + Locator: R.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: R.User{ID: "user1", Name: "user name"}, } _, err := b.Create(comment) assert.Nil(t, err) - svc := Service{Interface: b} + svc := DataStore{Interface: b} res, err := svc.Counts("radio-t", []string{"https://radio-t.com/2"}) assert.Nil(t, err) - assert.Equal(t, []PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res) + assert.Equal(t, []R.PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res) res, err = svc.Counts("radio-t", []string{"https://radio-t.com", "https://radio-t.com/2", "blah"}) assert.Nil(t, err) - assert.Equal(t, []PostInfo{ + assert.Equal(t, []R.PostInfo{ {URL: "https://radio-t.com", Count: 2}, {URL: "https://radio-t.com/2", Count: 1}, {URL: "blah", Count: 0}, }, res) } + +// makes new boltdb, put two records +func prepStoreEngine(t *testing.T) engine.Interface { + os.Remove(testDb) + + boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"}) + assert.Nil(t, err) + b := boltStore + + comment := R.Comment{ + ID: "id-1", + Text: `some text, link`, + Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: R.User{ID: "user1", Name: "user name"}, + } + _, err = b.Create(comment) + assert.Nil(t, err) + + comment = R.Comment{ + ID: "id-2", + Text: "some text2", + Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), + Locator: R.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: R.User{ID: "user1", Name: "user name"}, + } + _, err = b.Create(comment) + assert.Nil(t, err) + + return b +} diff --git a/app/store/user.go b/app/store/user.go new file mode 100644 index 00000000..42fe7be9 --- /dev/null +++ b/app/store/user.go @@ -0,0 +1,50 @@ +package store + +import ( + "crypto/hmac" + "crypto/sha1" + "fmt" + "hash/crc64" + "log" + "strconv" +) + +// User holds user-related info +type User struct { + Name string `json:"name"` + ID string `json:"id"` + Picture string `json:"picture"` + Admin bool `json:"admin"` + Blocked bool `json:"block,omitempty"` + IP string `json:"ip,omitempty"` +} + +// HashIP replace IP field with hashed hmac +func (u *User) HashIP(secret string) { + + hashVal := func(val string) string { + if _, err := strconv.ParseUint(val, 16, 64); err == nil || val == "" { + return val // already hashed + } + key := []byte(secret) + h := hmac.New(sha1.New, key) + if _, err := h.Write([]byte(val)); err != nil { + log.Printf("[WARN] can't hash ip, %s", err) + } + return fmt.Sprintf("%x", h.Sum(nil)) + } + + u.IP = hashVal(u.IP) +} + +// EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases +// we need hashing for parts of id, in some others hashing for non-User values. +func EncodeID(id string) string { + h := sha1.New() + if _, err := h.Write([]byte(id)); err != nil { + // fail back to crc64 + log.Printf("[WARN] can't hash id %s, %s", id, err) + return fmt.Sprintf("%x", crc64.Checksum([]byte(id), crc64.MakeTable(crc64.ECMA))) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/app/store/user_test.go b/app/store/user_test.go new file mode 100644 index 00000000..3bd21a12 --- /dev/null +++ b/app/store/user_test.go @@ -0,0 +1,42 @@ +package store + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUser_EncodeID(t *testing.T) { + tbl := []struct { + id string + hash string + }{ + {"myid", "6e34471f84557e1713012d64a7477c71bfdac631"}, + {"", "da39a3ee5e6b4b0d3255bfef95601890afd80709"}, + {"blah blah", "135a1e01bae742c4a576b20fd41a683f6483ca43"}, + } + + for i, tt := range tbl { + assert.Equal(t, tt.hash, EncodeID(tt.id), "case #%d", i) + } +} + +func TestUser_HashIP(t *testing.T) { + tbl := []struct { + ip string + hash1, hash2 string + }{ + {"127.0.0.1", "ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741", "dbc7c999343f003f189f70aaf52cc04443f90790"}, + {"8.8.8.8", "8cee77c27e32a2b5aec95c29888ac9946618d9a2", "70a46afce9633f010b06e129b8ad08243a1c4da9"}, + } + + for i, tt := range tbl { + u := User{IP: tt.ip} + u.HashIP("") + assert.Equal(t, tt.hash1, u.IP, "case #%d", i) + + u = User{IP: tt.ip} + u.HashIP("123456") + assert.Equal(t, tt.hash2, u.IP, "case #%d", i) + } +}