diff --git a/README.md b/README.md index a7810333..3c55a92e 100644 --- a/README.md +++ b/README.md @@ -477,3 +477,7 @@ _all admin calls require auth and admin privilege_ * User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse. * All avatars cached locally to prevent rate limiters from google/github/facebook/yandex. * Docker build uses [publicly available](https://github.com/umputun/baseimage) base images. + +## Current status + +Remark42 is still under development and the first stable v1.0 is about to be released. API (v1) is stable and won't have any breaking changes. diff --git a/app/main.go b/app/main.go index 668887d2..45c206dd 100644 --- a/app/main.go +++ b/app/main.go @@ -130,7 +130,7 @@ func New(opts Opts) (*Application, error) { MaxCommentSize: opts.MaxCommentSize, } - loadingCache, err := cache.NewLoadingCache(cache.MaxValSize(opts.MaxCachedValue), cache.MaxKeys(opts.MaxCachedItems), + loadingCache, err := cache.NewMemoryCache(cache.MaxValSize(opts.MaxCachedValue), cache.MaxKeys(opts.MaxCachedItems), cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port))) if err != nil { return nil, err diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 565c1f34..db752891 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -204,8 +204,9 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res [] for i, c := range comments { + blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) // process blocked users - if a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) { + if blocked { if !isAdmin { // reset comment to deleted for non-admins c.SetDeleted(store.SoftDelete) } @@ -213,6 +214,11 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res [] c.Deleted = true } + // set verified status retroactively + if !blocked { + c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID) + } + // hide info from non-admins if !isAdmin { c.User.IP = "" diff --git a/app/rest/api/admin_test.go b/app/rest/api/admin_test.go index f43ebe25..5a38c431 100644 --- a/app/rest/api/admin_test.go +++ b/app/rest/api/admin_test.go @@ -265,7 +265,7 @@ func TestAdmin_ReadOnly(t *testing.T) { assert.Nil(t, err) assert.True(t, info.ReadOnly) - // resset post's read-only + // reset post's read-only req, err = http.NewRequest(http.MethodPut, fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil) assert.Nil(t, err) @@ -305,6 +305,15 @@ func TestAdmin_Verify(t *testing.T) { verified = srv.DataService.IsVerified("radio-t", "user1") assert.True(t, verified) + res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time") + assert.Equal(t, 200, code) + comments := []store.Comment{} + err = json.Unmarshal([]byte(res), &comments) + assert.Nil(t, err) + assert.Equal(t, 2, len(comments), "should have 2 comments") + assert.Equal(t, "test test #1", comments[0].Text) + assert.True(t, comments[0].User.Verified) + req, err = http.NewRequest(http.MethodPut, fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil) assert.Nil(t, err) @@ -313,6 +322,16 @@ func TestAdmin_Verify(t *testing.T) { require.Nil(t, err) verified = srv.DataService.IsVerified("radio-t", "user1") assert.False(t, verified) + + res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time") + assert.Equal(t, 200, code) + comments = []store.Comment{} + err = json.Unmarshal([]byte(res), &comments) + assert.Nil(t, err) + assert.Equal(t, 2, len(comments), "should have 2 comments") + assert.Equal(t, "test test #1", comments[0].Text) + assert.False(t, comments[0].User.Verified) + } func TestAdmin_ExportStream(t *testing.T) { diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 7e69c36a..848e2c99 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -172,7 +172,7 @@ func (s *Rest) routes() chi.Router { for i := range allowed { allowed[i] = "Allow: /api/v1" + allowed[i] } - render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")) + render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n") }) // file server for static content from /web diff --git a/app/rest/api/rest_private.go b/app/rest/api/rest_private.go index dff8667c..c0fd9ce8 100644 --- a/app/rest/api/rest_private.go +++ b/app/rest/api/rest_private.go @@ -132,13 +132,18 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, res) } -// GET /user - returns user info +// GET /user?site=siteID - returns user info func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) { user, err := rest.GetUserInfo(r) if err != nil { rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") return } + + if siteID := r.URL.Query().Get("site"); siteID != "" { + user.Verified = s.DataService.IsVerified(siteID, user.ID) + } + render.JSON(w, r, user) } diff --git a/app/rest/api/rest_public_test.go b/app/rest/api/rest_public_test.go index 27654b31..3e68c712 100644 --- a/app/rest/api/rest_public_test.go +++ b/app/rest/api/rest_public_test.go @@ -436,5 +436,5 @@ func TestRest_Robots(t *testing.T) { assert.Equal(t, 200, code) assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+ "Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+ - "Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar", string(body)) + "Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body)) } diff --git a/app/rest/cache/cache.go b/app/rest/cache/cache.go index 8ac138c1..f734f42f 100644 --- a/app/rest/cache/cache.go +++ b/app/rest/cache/cache.go @@ -1,12 +1,9 @@ package cache import ( - "log" "net/http" "strings" - "sync/atomic" - "github.com/hashicorp/golang-lru" "github.com/pkg/errors" "github.com/umputun/remark/app/rest" ) @@ -22,7 +19,8 @@ func Key(key string, scopes ...string) string { return strings.Join(scopes, "$$") + "@@" + key } -func parseKey(fullKey string) (key string, scopes []string, err error) { +// ParseKey gets compound key created by Key func and split it to the actual key and scopes +func ParseKey(fullKey string) (key string, scopes []string, err error) { elems := strings.Split(fullKey, "@@") if len(elems) != 2 { return "", nil, errors.Errorf("can't parse cache key %s", key) @@ -35,116 +33,6 @@ func parseKey(fullKey string) (key string, scopes []string, err error) { return key, scopes, nil } -// loadingCache implements LoadingCache interface on top of cache.Cache (go-cache) -type loadingCache struct { - bytesCache *lru.Cache - postFlushFn func() - maxKeys int - maxValueSize int - maxCacheSize int64 - currentSize int64 -} - -// NewLoadingCache makes loadingCache implementation -func NewLoadingCache(options ...Option) (LoadingCache, error) { - res := loadingCache{ - postFlushFn: func() {}, - maxKeys: 1000, - maxValueSize: 0, - } - for _, opt := range options { - if err := opt(&res); err != nil { - log.Printf("[WARN] failed to set cache option, %v", err) - } - } - - onEvicted := func(key interface{}, value interface{}) { - size := len(value.([]byte)) - atomic.AddInt64(&res.currentSize, -1*int64(size)) - } - - var err error - // OnEvicted called automatically for expired and manually deleted - if res.bytesCache, err = lru.NewWithEvict(res.maxKeys, onEvicted); err != nil { - return nil, errors.Wrap(err, "failed to make cache") - } - - log.Printf("[DEBUG] create lru cache, maxKeys=%d, maxValueSize=%d", res.maxKeys, res.maxValueSize) - return &res, nil -} - -// Get is loading cache method to get value by key or load via fn if not found -func (lc *loadingCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) { - if b, ok := lc.bytesCache.Get(key); ok { - return b.([]byte), nil - } - - if data, err = fn(); err != nil { - return data, err - } - if lc.allowed(data) { - lc.bytesCache.Add(key, data) - atomic.AddInt64(&lc.currentSize, int64(len(data))) - - if lc.maxCacheSize > 0 && atomic.LoadInt64(&lc.currentSize) > lc.maxCacheSize { - for atomic.LoadInt64(&lc.currentSize) > lc.maxCacheSize { - lc.bytesCache.RemoveOldest() - } - } - } - return data, nil -} - -// Flush clears cache and calls postFlushFn async -func (lc *loadingCache) Flush(scopes ...string) { - - if len(scopes) == 0 { - lc.bytesCache.Purge() - go lc.postFlushFn() - return - } - - // check if fullKey has matching scopes - inScope := func(fullKey string) bool { - for _, s := range scopes { - _, keyScopes, err := parseKey(fullKey) - if err != nil { - return false - } - for _, ks := range keyScopes { - if ks == s { - return true - } - } - } - return false - } - - // all matchedKeys should be collected first - // we can't delete it from locked section, it will lock on eviction callback - matchedKeys := []string{} - for _, k := range lc.bytesCache.Keys() { - key := k.(string) - if inScope(key) { - matchedKeys = append(matchedKeys, key) - } - } - for _, mkey := range matchedKeys { - lc.bytesCache.Remove(mkey) - } - - if lc.postFlushFn != nil { - go lc.postFlushFn() - } -} - -func (lc *loadingCache) allowed(data []byte) bool { - if lc.maxValueSize > 0 && len(data) >= lc.maxValueSize { - return false - } - return true -} - // URLKey gets url from request to use it as cache key // admins will have different keys in order to prevent leak of admin-only data to regular users func URLKey(r *http.Request) string { diff --git a/app/rest/cache/cache_test.go b/app/rest/cache/cache_test.go index 3f1f443f..22cabca0 100644 --- a/app/rest/cache/cache_test.go +++ b/app/rest/cache/cache_test.go @@ -1,187 +1,42 @@ package cache import ( - "errors" - "fmt" - "math/rand" "net/http" - "sync" - "sync/atomic" "testing" - "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/umputun/remark/app/rest" "github.com/umputun/remark/app/store" ) -func TestLoadingCache_Get(t *testing.T) { - var postFnCall, coldCalls int32 - lc, err := NewLoadingCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) })) - require.Nil(t, err) - res, err := lc.Get("key", func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte("result"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result", string(res)) - assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) +func TestCache_Keys(t *testing.T) { + tbl := []struct { + key string + scopes []string + full string + }{ + {"key1", []string{"s1"}, "s1@@key1"}, + {"key2", []string{"s11", "s2"}, "s11$$s2@@key2"}, + {"key3", []string{}, "@@key3"}, + } - res, err = lc.Get("key", func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte("result"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result", string(res)) - assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + for n, tt := range tbl { + full := Key(tt.key, tt.scopes...) + assert.Equal(t, tt.full, full, "making key, #%d", n) - lc.Flush() - time.Sleep(100 * time.Millisecond) // let postFn to do its thing - assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall)) - - _, err = lc.Get("key", func() ([]byte, error) { - return nil, errors.New("err") - }) - assert.NotNil(t, err) -} - -func TestLoadingCache_MaxKeys(t *testing.T) { - var postFnCall, coldCalls int32 - lc, err := NewLoadingCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }), - MaxKeys(5), MaxValSize(10)) - require.Nil(t, err) - - // put 5 keys to cache - for i := 0; i < 5; i++ { - res, e := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte(fmt.Sprintf("result-%d", i)), nil - }) + k, s, e := ParseKey(full) assert.Nil(t, e) - assert.Equal(t, fmt.Sprintf("result-%d", i), string(res)) - assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + assert.Equal(t, tt.scopes, s) + assert.Equal(t, tt.key, k) } - // check if really cached - res, err := lc.Get("key-3", func() ([]byte, error) { - return []byte("result-blah"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-3", string(res), "should be cached") - - // try to cache after maxKeys reached - res, err = lc.Get("key-X", func() ([]byte, error) { - return []byte("result-X"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-X", string(res)) - - assert.Equal(t, 5, lc.(*loadingCache).bytesCache.Len()) - - // put to cache and make sure it cached - res, err = lc.Get("key-Z", func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - - res, err = lc.Get("key-Z", func() ([]byte, error) { - return []byte("result-Zzzz"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res), "got cached value") - assert.Equal(t, 5, lc.(*loadingCache).bytesCache.Len()) + _, _, err := ParseKey("abc") + assert.Error(t, err) + _, _, err = ParseKey("") + assert.Error(t, err) } -func TestLoadingCache_MaxValueSize(t *testing.T) { - lc, err := NewLoadingCache(MaxKeys(5), MaxValSize(10)) - require.Nil(t, err) - // put good size value to cache and make sure it cached - res, err := lc.Get("key-Z", func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - - res, err = lc.Get("key-Z", func() ([]byte, error) { - return []byte("result-Zzzz"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res), "got cached value") - - // put too big value to cache and make sure it is not cached - res, err = lc.Get("key-Big", func() ([]byte, error) { - return []byte("1234567890"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "1234567890", string(res)) - - res, err = lc.Get("key-Big", func() ([]byte, error) { - return []byte("result-big"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-big", string(res), "got not cached value") -} - -func TestLoadingCache_MaxCacheSize(t *testing.T) { - lc, err := NewLoadingCache(MaxKeys(50), MaxCacheSize(20)) - require.Nil(t, err) - - // put good size value to cache and make sure it cached - res, err := lc.Get("key-Z", func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - assert.Equal(t, int64(8), lc.(*loadingCache).currentSize) - - _, err = lc.Get("key-Z2", func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, int64(16), lc.(*loadingCache).currentSize) - - // this will cause removal - _, err = lc.Get("key-Z3", func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, int64(16), lc.(*loadingCache).currentSize) - - assert.Equal(t, 2, lc.(*loadingCache).bytesCache.Len()) -} - -func TestLoadingCache_MaxCacheSizeParallel(t *testing.T) { - lc, err := NewLoadingCache(MaxCacheSize(123), MaxKeys(10000)) - require.Nil(t, err) - - wg := sync.WaitGroup{} - for i := 0; i < 1000; i++ { - wg.Add(1) - i := i - go func() { - time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond) - defer wg.Done() - res, err := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) { - return []byte(fmt.Sprintf("result-%d", i)), nil - }) - require.Nil(t, err) - require.Equal(t, fmt.Sprintf("result-%d", i), string(res)) - size := atomic.LoadInt64(&lc.(*loadingCache).currentSize) - require.True(t, size < 200 && size >= 0, "unexpected size=%d", size) // won't be exactly 123 due parallel - }() - } - wg.Wait() - assert.True(t, lc.(*loadingCache).currentSize < 123 && lc.(*loadingCache).currentSize >= 0) - t.Log("size=", lc.(*loadingCache).currentSize) -} - -func TestLoadingCache_URLKey(t *testing.T) { +func TestCache_URLKey(t *testing.T) { r, err := http.NewRequest("GET", "http://blah/123", nil) assert.Nil(t, err) key := URLKey(r) @@ -197,151 +52,3 @@ func TestLoadingCache_URLKey(t *testing.T) { key = URLKey(r) assert.Equal(t, "admin!!http://blah/123?key=v&k2=v2", key) } - -func TestLoadingCache_Parallel(t *testing.T) { - var coldCalls int32 - lc, err := NewLoadingCache() - require.Nil(t, err) - - res, err := lc.Get("key", func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(res)) - - wg := sync.WaitGroup{} - for i := 0; i < 1000; i++ { - wg.Add(1) - i := i - go func() { - defer wg.Done() - res, err := lc.Get("key", func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte(fmt.Sprintf("result-%d", i)), nil - }) - require.Nil(t, err) - require.Equal(t, "value", string(res)) - }() - } - wg.Wait() - assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls)) -} - -func TestLoadingCache_Scopes(t *testing.T) { - lc, err := NewLoadingCache() - require.Nil(t, err) - - res, err := lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(res)) - - res, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) { - return []byte("value2"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value2", string(res)) - - assert.Equal(t, 2, lc.(*loadingCache).bytesCache.Len()) - lc.Flush("s1") - assert.Equal(t, 1, lc.(*loadingCache).bytesCache.Len()) - - lc.Get(Key("key2", "s2"), func() ([]byte, error) { - assert.Fail(t, "should stay") - return nil, nil - }) - - res, err = lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) { - return []byte("value-upd"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value-upd", string(res), "was deleted, update") -} - -func TestLoadingCache_Flush(t *testing.T) { - lc, err := NewLoadingCache() - require.Nil(t, err) - - addToCache := func(key string, scopes ...string) { - res, err := lc.Get(key, func() ([]byte, error) { - return []byte("value" + key), nil - }) - require.Nil(t, err) - require.Equal(t, "value"+key, string(res)) - } - - init := func() { - lc.Flush() - addToCache(Key("key1", "s1", "s2")) - addToCache(Key("key2", "s1", "s2", "s3")) - addToCache(Key("key3", "s1", "s2", "s3")) - addToCache(Key("key4", "s2", "s3")) - addToCache(Key("key5", "s2")) - addToCache(Key("key6")) - addToCache(Key("key7", "s4", "s3")) - require.Equal(t, 7, lc.(*loadingCache).bytesCache.Len(), "cache init") - } - - tbl := []struct { - scopes []string - left int - msg string - }{ - {[]string{}, 0, "full flush, no scopes"}, - {[]string{"s0"}, 7, "flush wrong scope"}, - {[]string{"s1"}, 4, "flush s1 scope"}, - {[]string{"s2", "s1"}, 2, "flush s2+s1 scope"}, - {[]string{"s1", "s2"}, 2, "flush s1+s2 scope"}, - {[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"}, - {[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"}, - {[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"}, - } - - for i, tt := range tbl { - init() - lc.Flush(tt.scopes...) - assert.Equal(t, tt.left, lc.(*loadingCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i) - } -} - -func TestLoadingCache_FlushFailed(t *testing.T) { - lc, err := NewLoadingCache() - require.Nil(t, err) - val, err := lc.Get("invalid-composite", func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(val)) - assert.Equal(t, 1, lc.(*loadingCache).bytesCache.Len()) - - lc.Flush("invalid-composite") - assert.Equal(t, 1, lc.(*loadingCache).bytesCache.Len()) -} - -func TestLoadingCache_Keys(t *testing.T) { - tbl := []struct { - key string - scopes []string - full string - }{ - {"key1", []string{"s1"}, "s1@@key1"}, - {"key2", []string{"s11", "s2"}, "s11$$s2@@key2"}, - {"key3", []string{}, "@@key3"}, - } - - for n, tt := range tbl { - full := Key(tt.key, tt.scopes...) - assert.Equal(t, tt.full, full, "making key, #%d", n) - - k, s, e := parseKey(full) - assert.Nil(t, e) - assert.Equal(t, tt.scopes, s) - assert.Equal(t, tt.key, k) - } - - _, _, err := parseKey("abc") - assert.Error(t, err) - _, _, err = parseKey("") - assert.Error(t, err) -} diff --git a/app/rest/cache/memory.go b/app/rest/cache/memory.go new file mode 100644 index 00000000..56511ae3 --- /dev/null +++ b/app/rest/cache/memory.go @@ -0,0 +1,119 @@ +package cache + +import ( + "log" + "sync/atomic" + + "github.com/hashicorp/golang-lru" + "github.com/pkg/errors" +) + +// memoryCache implements LoadingCache interface on top of cache.Cache (go-cache) +type memoryCache struct { + bytesCache *lru.Cache + postFlushFn func() + maxKeys int + maxValueSize int + maxCacheSize int64 + currentSize int64 +} + +// NewMemoryCache makes memoryCache implementation +func NewMemoryCache(options ...Option) (LoadingCache, error) { + res := memoryCache{ + postFlushFn: func() {}, + maxKeys: 1000, + maxValueSize: 0, + } + for _, opt := range options { + if err := opt(&res); err != nil { + log.Printf("[WARN] failed to set cache option, %v", err) + } + } + + onEvicted := func(key interface{}, value interface{}) { + size := len(value.([]byte)) + atomic.AddInt64(&res.currentSize, -1*int64(size)) + } + + var err error + // OnEvicted called automatically for expired and manually deleted + if res.bytesCache, err = lru.NewWithEvict(res.maxKeys, onEvicted); err != nil { + return nil, errors.Wrap(err, "failed to make cache") + } + + log.Printf("[DEBUG] create lru cache, maxKeys=%d, maxValueSize=%d", res.maxKeys, res.maxValueSize) + return &res, nil +} + +// Get is loading cache method to get value by key or load via fn if not found +func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) { + if b, ok := m.bytesCache.Get(key); ok { + return b.([]byte), nil + } + + if data, err = fn(); err != nil { + return data, err + } + if m.allowed(data) { + m.bytesCache.Add(key, data) + atomic.AddInt64(&m.currentSize, int64(len(data))) + + if m.maxCacheSize > 0 && atomic.LoadInt64(&m.currentSize) > m.maxCacheSize { + for atomic.LoadInt64(&m.currentSize) > m.maxCacheSize { + m.bytesCache.RemoveOldest() + } + } + } + return data, nil +} + +// Flush clears cache and calls postFlushFn async +func (m *memoryCache) Flush(scopes ...string) { + + if len(scopes) == 0 { + m.bytesCache.Purge() + go m.postFlushFn() + return + } + + // check if fullKey has matching scopes + inScope := func(fullKey string) bool { + for _, s := range scopes { + _, keyScopes, err := ParseKey(fullKey) + if err != nil { + return false + } + for _, ks := range keyScopes { + if ks == s { + return true + } + } + } + return false + } + + // all matchedKeys should be collected first + // we can't delete it from locked section, it will lock on eviction callback + matchedKeys := []string{} + for _, k := range m.bytesCache.Keys() { + key := k.(string) + if inScope(key) { + matchedKeys = append(matchedKeys, key) + } + } + for _, mkey := range matchedKeys { + m.bytesCache.Remove(mkey) + } + + if m.postFlushFn != nil { + go m.postFlushFn() + } +} + +func (m *memoryCache) allowed(data []byte) bool { + if m.maxValueSize > 0 && len(data) >= m.maxValueSize { + return false + } + return true +} diff --git a/app/rest/cache/memory_test.go b/app/rest/cache/memory_test.go new file mode 100644 index 00000000..b4b5f99b --- /dev/null +++ b/app/rest/cache/memory_test.go @@ -0,0 +1,299 @@ +package cache + +import ( + "fmt" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMemoryCache_Get(t *testing.T) { + var postFnCall, coldCalls int32 + lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) })) + require.Nil(t, err) + res, err := lc.Get("key", func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte("result"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result", string(res)) + assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) + assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + + res, err = lc.Get("key", func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte("result"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result", string(res)) + assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) + assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + + lc.Flush() + time.Sleep(100 * time.Millisecond) // let postFn to do its thing + assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall)) + + _, err = lc.Get("key", func() ([]byte, error) { + return nil, errors.New("err") + }) + assert.NotNil(t, err) +} + +func TestMemoryCache_MaxKeys(t *testing.T) { + var postFnCall, coldCalls int32 + lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }), + MaxKeys(5), MaxValSize(10)) + require.Nil(t, err) + + // put 5 keys to cache + for i := 0; i < 5; i++ { + res, e := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte(fmt.Sprintf("result-%d", i)), nil + }) + assert.Nil(t, e) + assert.Equal(t, fmt.Sprintf("result-%d", i), string(res)) + assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls)) + assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + } + + // check if really cached + res, err := lc.Get("key-3", func() ([]byte, error) { + return []byte("result-blah"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-3", string(res), "should be cached") + + // try to cache after maxKeys reached + res, err = lc.Get("key-X", func() ([]byte, error) { + return []byte("result-X"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-X", string(res)) + + assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len()) + + // put to cache and make sure it cached + res, err = lc.Get("key-Z", func() ([]byte, error) { + return []byte("result-Z"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res)) + + res, err = lc.Get("key-Z", func() ([]byte, error) { + return []byte("result-Zzzz"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res), "got cached value") + assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len()) +} + +func TestMemoryCache_MaxValueSize(t *testing.T) { + lc, err := NewMemoryCache(MaxKeys(5), MaxValSize(10)) + require.Nil(t, err) + // put good size value to cache and make sure it cached + res, err := lc.Get("key-Z", func() ([]byte, error) { + return []byte("result-Z"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res)) + + res, err = lc.Get("key-Z", func() ([]byte, error) { + return []byte("result-Zzzz"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res), "got cached value") + + // put too big value to cache and make sure it is not cached + res, err = lc.Get("key-Big", func() ([]byte, error) { + return []byte("1234567890"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "1234567890", string(res)) + + res, err = lc.Get("key-Big", func() ([]byte, error) { + return []byte("result-big"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-big", string(res), "got not cached value") +} + +func TestMemoryCache_MaxCacheSize(t *testing.T) { + lc, err := NewMemoryCache(MaxKeys(50), MaxCacheSize(20)) + require.Nil(t, err) + + // put good size value to cache and make sure it cached + res, err := lc.Get("key-Z", func() ([]byte, error) { + return []byte("result-Z"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res)) + assert.Equal(t, int64(8), lc.(*memoryCache).currentSize) + + _, err = lc.Get("key-Z2", func() ([]byte, error) { + return []byte("result-Z"), nil + }) + assert.Nil(t, err) + assert.Equal(t, int64(16), lc.(*memoryCache).currentSize) + + // this will cause removal + _, err = lc.Get("key-Z3", func() ([]byte, error) { + return []byte("result-Z"), nil + }) + assert.Nil(t, err) + assert.Equal(t, int64(16), lc.(*memoryCache).currentSize) + + assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len()) +} + +func TestMemoryCache_MaxCacheSizeParallel(t *testing.T) { + lc, err := NewMemoryCache(MaxCacheSize(123), MaxKeys(10000)) + require.Nil(t, err) + + wg := sync.WaitGroup{} + for i := 0; i < 1000; i++ { + wg.Add(1) + i := i + go func() { + time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond) + defer wg.Done() + res, err := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) { + return []byte(fmt.Sprintf("result-%d", i)), nil + }) + require.Nil(t, err) + require.Equal(t, fmt.Sprintf("result-%d", i), string(res)) + size := atomic.LoadInt64(&lc.(*memoryCache).currentSize) + require.True(t, size < 200 && size >= 0, "unexpected size=%d", size) // won't be exactly 123 due parallel + }() + } + wg.Wait() + assert.True(t, lc.(*memoryCache).currentSize < 123 && lc.(*memoryCache).currentSize >= 0) + t.Log("size=", lc.(*memoryCache).currentSize) +} + +func TestMemoryCache_Parallel(t *testing.T) { + var coldCalls int32 + lc, err := NewMemoryCache() + require.Nil(t, err) + + res, err := lc.Get("key", func() ([]byte, error) { + return []byte("value"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value", string(res)) + + wg := sync.WaitGroup{} + for i := 0; i < 1000; i++ { + wg.Add(1) + i := i + go func() { + defer wg.Done() + res, err := lc.Get("key", func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte(fmt.Sprintf("result-%d", i)), nil + }) + require.Nil(t, err) + require.Equal(t, "value", string(res)) + }() + } + wg.Wait() + assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls)) +} + +func TestMemoryCache_Scopes(t *testing.T) { + lc, err := NewMemoryCache() + require.Nil(t, err) + + res, err := lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) { + return []byte("value"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value", string(res)) + + res, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) { + return []byte("value2"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value2", string(res)) + + assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len()) + lc.Flush("s1") + assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) + + lc.Get(Key("key2", "s2"), func() ([]byte, error) { + assert.Fail(t, "should stay") + return nil, nil + }) + + res, err = lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) { + return []byte("value-upd"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value-upd", string(res), "was deleted, update") +} + +func TestMemoryCache_Flush(t *testing.T) { + lc, err := NewMemoryCache() + require.Nil(t, err) + + addToCache := func(key string, scopes ...string) { + res, err := lc.Get(key, func() ([]byte, error) { + return []byte("value" + key), nil + }) + require.Nil(t, err) + require.Equal(t, "value"+key, string(res)) + } + + init := func() { + lc.Flush() + addToCache(Key("key1", "s1", "s2")) + addToCache(Key("key2", "s1", "s2", "s3")) + addToCache(Key("key3", "s1", "s2", "s3")) + addToCache(Key("key4", "s2", "s3")) + addToCache(Key("key5", "s2")) + addToCache(Key("key6")) + addToCache(Key("key7", "s4", "s3")) + require.Equal(t, 7, lc.(*memoryCache).bytesCache.Len(), "cache init") + } + + tbl := []struct { + scopes []string + left int + msg string + }{ + {[]string{}, 0, "full flush, no scopes"}, + {[]string{"s0"}, 7, "flush wrong scope"}, + {[]string{"s1"}, 4, "flush s1 scope"}, + {[]string{"s2", "s1"}, 2, "flush s2+s1 scope"}, + {[]string{"s1", "s2"}, 2, "flush s1+s2 scope"}, + {[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"}, + {[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"}, + {[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"}, + } + + for i, tt := range tbl { + init() + lc.Flush(tt.scopes...) + assert.Equal(t, tt.left, lc.(*memoryCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i) + } +} + +func TestMemoryCache_FlushFailed(t *testing.T) { + lc, err := NewMemoryCache() + require.Nil(t, err) + val, err := lc.Get("invalid-composite", func() ([]byte, error) { + return []byte("value"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value", string(val)) + assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) + + lc.Flush("invalid-composite") + assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) +} diff --git a/app/rest/cache/options.go b/app/rest/cache/options.go index e077cf8f..2902b0fa 100644 --- a/app/rest/cache/options.go +++ b/app/rest/cache/options.go @@ -1,12 +1,12 @@ package cache // Option func type -type Option func(lc *loadingCache) error +type Option func(lc *memoryCache) error // MaxValSize functional option defines the largest value's size allowed to be cached // By default it is 0, which means unlimited. func MaxValSize(max int) Option { - return func(lc *loadingCache) error { + return func(lc *memoryCache) error { lc.maxValueSize = max return nil } @@ -15,7 +15,7 @@ func MaxValSize(max int) Option { // MaxKeys functional option defines how many keys to keep. // By default it is 0, which means unlimited. func MaxKeys(max int) Option { - return func(lc *loadingCache) error { + return func(lc *memoryCache) error { lc.maxKeys = max return nil } @@ -24,7 +24,7 @@ func MaxKeys(max int) Option { // MaxCacheSize functional option defines the total size of cached data. // By default it is 0, which means unlimited. func MaxCacheSize(max int64) Option { - return func(lc *loadingCache) error { + return func(lc *memoryCache) error { lc.maxCacheSize = max return nil } @@ -32,7 +32,7 @@ func MaxCacheSize(max int64) Option { // PostFlushFn functional option defines how callback function called after each Flush. func PostFlushFn(postFlushFn func()) Option { - return func(lc *loadingCache) error { + return func(lc *memoryCache) error { lc.postFlushFn = postFlushFn return nil } diff --git a/web/app/common/api.js b/web/app/common/api.js index de15804c..bd1007eb 100644 --- a/web/app/common/api.js +++ b/web/app/common/api.js @@ -72,6 +72,16 @@ export const unpin = ({ id, url }) => fetcher.put({ withCredentials: true, }); +export const verify = ({ id }) => fetcher.put({ + url: `/admin/verify/${id}?verified=1`, + withCredentials: true, +}); + +export const unverify = ({ id }) => fetcher.put({ + url: `/admin/verify/${id}?verified=0`, + withCredentials: true, +}); + export const remove = ({ id }) => fetcher.delete({ url: `/admin/comment/${id}?url=${url}`, withCredentials: true, @@ -107,6 +117,8 @@ export default { pin, unpin, + verify, + unverify, remove, blockUser, unblockUser, diff --git a/web/app/components/comment/__verification/_active/comment__verification_active.scss b/web/app/components/comment/__verification/_active/comment__verification_active.scss new file mode 100644 index 00000000..e274b88a --- /dev/null +++ b/web/app/components/comment/__verification/_active/comment__verification_active.scss @@ -0,0 +1,3 @@ +.comment__verification_active { + background-image: url('comment__verification_active.svg'); +} diff --git a/web/app/components/comment/__verification/_active/comment__verification_active.svg b/web/app/components/comment/__verification/_active/comment__verification_active.svg new file mode 100644 index 00000000..117e912e --- /dev/null +++ b/web/app/components/comment/__verification/_active/comment__verification_active.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/web/app/components/comment/__verification/_clickable/comment__verification_clickable.scss b/web/app/components/comment/__verification/_clickable/comment__verification_clickable.scss new file mode 100644 index 00000000..c406abfc --- /dev/null +++ b/web/app/components/comment/__verification/_clickable/comment__verification_clickable.scss @@ -0,0 +1,3 @@ +.comment__verification_clickable { + cursor: pointer; +} diff --git a/web/app/components/comment/__verification/comment__verification.scss b/web/app/components/comment/__verification/comment__verification.scss new file mode 100644 index 00000000..79b65086 --- /dev/null +++ b/web/app/components/comment/__verification/comment__verification.scss @@ -0,0 +1,13 @@ +.comment__verification { + display: inline-block; + width: 12px; + height: 12px; + margin-left: 4px; + vertical-align: middle; + background: url('comment__verification.svg') center no-repeat; + background-size: cover; + + &:hover { + opacity: .75; + } +} diff --git a/web/app/components/comment/__verification/comment__verification.svg b/web/app/components/comment/__verification/comment__verification.svg new file mode 100644 index 00000000..aafbd798 --- /dev/null +++ b/web/app/components/comment/__verification/comment__verification.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/web/app/components/comment/comment.jsx b/web/app/components/comment/comment.jsx index 65f4a043..d36a2eab 100644 --- a/web/app/components/comment/comment.jsx +++ b/web/app/components/comment/comment.jsx @@ -15,6 +15,7 @@ export default class Comment extends Component { isReplying: false, isEditing: false, isUserIdVisible: false, + isUserVerified: false, editTimeLeft: null, }; @@ -33,6 +34,8 @@ export default class Comment extends Component { this.onReply = this.onReply.bind(this); this.onPinClick = this.onPinClick.bind(this); this.onUnpinClick = this.onUnpinClick.bind(this); + this.onVerifyClick = this.onVerifyClick.bind(this); + this.onUnverifyClick = this.onUnverifyClick.bind(this); this.onBlockClick = this.onBlockClick.bind(this); this.onUnblockClick = this.onUnblockClick.bind(this); this.onDeleteClick = this.onDeleteClick.bind(this); @@ -151,6 +154,30 @@ export default class Comment extends Component { } } + onVerifyClick() { + const { id, user: { id: userId } } = this.props.data; + + if (confirm('Do you want to verify this user?')) { + this.setState({ isUserVerified: true }); + + api.verify({ id: userId }).then(() => { + api.getComment({ id }).then(comment => store.replaceComment(comment)); + }); + } + } + + onUnverifyClick() { + const { id, user: { id: userId } } = this.props.data; + + if (confirm('Do you want to unverify this user?')) { + this.setState({ isUserVerified: false }); + + api.unverify({ id: userId }).then(() => { + api.getComment({ id }).then(comment => store.replaceComment(comment)); + }); + } + } + onBlockClick() { const { id, user: { id: userId } } = this.props.data; @@ -272,7 +299,20 @@ export default class Comment extends Component { } } - render(props, { guest, isUserIdVisible, userBlocked, pinned, score, scoreIncreased, scoreDecreased, deleted, isReplying, isEditing, editTimeLeft }) { + render(props, { + guest, + isUserIdVisible, + userBlocked, + pinned, + score, + scoreIncreased, + scoreDecreased, + deleted, + isReplying, + isEditing, + isUserVerified, + editTimeLeft, + }) { const { data, mods = {} } = props; const isAdmin = !guest && store.get('user').admin; const isGuest = guest || !Object.keys(store.get('user')).length; @@ -314,6 +354,7 @@ export default class Comment extends Component { ...data.user, picture: data.user.picture.indexOf(API_BASE) === 0 ? `${BASE_URL}${data.user.picture}` : data.user.picture, isDefaultPicture: !data.user.picture.length, + verified: data.user.verified || isUserVerified, }, }; @@ -364,6 +405,26 @@ export default class Comment extends Component { isUserIdVisible && ({o.user.id}) } + { + isAdmin && ( + + ) + } + + { + !isAdmin && !!o.user.verified && ( + + ) + } + {o.time} { diff --git a/web/app/components/comment/index.js b/web/app/components/comment/index.js index 96b755ae..c2b83915 100644 --- a/web/app/components/comment/index.js +++ b/web/app/components/comment/index.js @@ -28,6 +28,10 @@ require('./__time/comment__time.scss'); require('./__user-id/comment__user-id.scss'); require('./__username/comment__username.scss'); +require('./__verification/comment__verification.scss'); +require('./__verification/_active/comment__verification_active.scss'); +require('./__verification/_clickable/comment__verification_clickable.scss'); + require('./__vote/comment__vote.scss'); require('./__vote/_disabled/comment__vote_disabled.scss'); require('./__vote/_selected/comment__vote_selected.scss');