mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cache to separate package

Browse Source
This commit is contained in:
Umputun
2018-05-25 02:08:54 -05:00
parent d8fee735f1
commit 08be684d76
7 changed files with 48 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.go
+3 -3
View File
@@ -19,9 +19,9 @@ import (
"github.com/umputun/remark/app/store/service"
"github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/rest"
"github.com/umputun/remark/app/rest/api"
"github.com/umputun/remark/app/rest/auth"
"github.com/umputun/remark/app/rest/cache"
"github.com/umputun/remark/app/rest/proxy"
)
@@ -114,8 +114,8 @@ func New(opts Opts) (*Application, error) {
MaxCommentSize: opts.MaxCommentSize,
}
cache := rest.NewLoadingCache(rest.MaxValSize(opts.MaxCachedValue), rest.MaxKeys(opts.MaxCachedItems),
rest.PostFlushFn(postFlushFn(opts.Sites, opts.Port)))
cache := cache.NewLoadingCache(cache.MaxValSize(opts.MaxCachedValue), cache.MaxKeys(opts.MaxCachedItems),
cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port)))
jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour)
app/rest/api/admin.go
+2 -1
View File
@@ -13,6 +13,7 @@ import (
"github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/rest"
"github.com/umputun/remark/app/rest/cache"
"github.com/umputun/remark/app/store"
"github.com/umputun/remark/app/store/service"
)
@@ -21,7 +22,7 @@ import (
type admin struct {
dataService service.DataStore
exporter migrator.Exporter
cache rest.LoadingCache
cache cache.LoadingCache
defAvatarURL string
}
app/rest/api/import.go
+2 -1
View File
@@ -17,12 +17,13 @@ import (
"github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/rest"
"github.com/umputun/remark/app/rest/cache"
)
// Import rest runs on unexposed port and available for local requests only
type Import struct {
Version string
Cache rest.LoadingCache
Cache cache.LoadingCache
NativeImporter migrator.Importer
DisqusImporter migrator.Importer
SecretKey string
app/rest/api/rest.go
+8 -7
View File
@@ -25,6 +25,7 @@ import (
"github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/rest"
"github.com/umputun/remark/app/rest/auth"
"github.com/umputun/remark/app/rest/cache"
"github.com/umputun/remark/app/rest/proxy"
"github.com/umputun/remark/app/store"
"github.com/umputun/remark/app/store/service"
@@ -36,7 +37,7 @@ type Rest struct {
DataService service.DataStore
Authenticator auth.Authenticator
Exporter migrator.Exporter
Cache rest.LoadingCache
Cache cache.LoadingCache
AvatarProxy *proxy.Avatar
ImageProxy *proxy.Image
WebRoot string
@@ -308,7 +309,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
}
log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), locator.SiteID, locator.URL), 4*time.Hour, func() ([]byte, error) {
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), 4*time.Hour, func() ([]byte, error) {
comments, e := s.DataService.Find(locator, sort)
if e != nil {
return nil, e
@@ -341,7 +342,7 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
limit = 0
}
data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), "last", siteID), 4*time.Hour, func() ([]byte, error) {
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), "last", siteID), 4*time.Hour, func() ([]byte, error) {
comments, e := s.DataService.Last(siteID, limit)
if e != nil {
return nil, e
@@ -404,7 +405,7 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), userID, siteID), 4*time.Hour, func() ([]byte, error) {
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), userID, siteID), 4*time.Hour, func() ([]byte, error) {
comments, count, e := s.DataService.User(siteID, userID, limit)
if e != nil {
return nil, e
@@ -485,7 +486,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
}
// key could be long for multiple posts, make it sha1
key := rest.URLKey(r) + strings.Join(posts, ",")
key := cache.URLKey(r) + strings.Join(posts, ",")
hasher := sha1.New()
if _, err := hasher.Write([]byte(key)); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
@@ -493,7 +494,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
}
sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
data, err := s.Cache.Get(rest.CacheKey(sha, siteID), 8*time.Hour, func() ([]byte, error) {
data, err := s.Cache.Get(cache.Key(sha, siteID), 8*time.Hour, func() ([]byte, error) {
counts, e := s.DataService.Counts(siteID, posts)
if e != nil {
return nil, e
@@ -521,7 +522,7 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
skip = v
}
data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), siteID), 8*time.Hour, func() ([]byte, error) {
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), 8*time.Hour, func() ([]byte, error) {
posts, e := s.DataService.List(siteID, limit, skip)
if e != nil {
return nil, e
app/rest/api/rss.go
+6 -4
View File
@@ -11,6 +11,7 @@ import (
"github.com/gorilla/feeds"
"github.com/umputun/remark/app/rest"
"github.com/umputun/remark/app/rest/cache"
"github.com/umputun/remark/app/store"
)
@@ -32,7 +33,7 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
sort := "-time"
log.Printf("[DEBUG] get rss for post %+v", locator)
data, err := s.Cache.Get(rest.URLKey(r), 4*time.Hour, func() ([]byte, error) {
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), 4*time.Hour, func() ([]byte, error) {
comments, e := s.DataService.Find(locator, sort)
if e != nil {
return nil, e
@@ -61,10 +62,11 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
// GET /rss/site?site=siteID
func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
log.Printf("[DEBUG] get rss for site %s", r.URL.Query().Get("site"))
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get rss for site %s", siteID)
data, err := s.Cache.Get(rest.URLKey(r), 4*time.Hour, func() ([]byte, error) {
comments, e := s.DataService.Last(r.URL.Query().Get("site"), maxRssItems)
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), 4*time.Hour, func() ([]byte, error) {
comments, e := s.DataService.Last(siteID, maxRssItems)
if e != nil {
return nil, e
}
app/rest/cache.go → app/rest/cache/cache.go
Vendored
+19 -15
View File
@@ -1,4 +1,4 @@
package rest
package cache
import (
"log"
@@ -9,6 +9,7 @@ import (
"github.com/patrickmn/go-cache"
"github.com/pkg/errors"
"github.com/umputun/remark/app/rest"
)
// LoadingCache defines interface for caching
@@ -17,8 +18,8 @@ type LoadingCache interface {
Flush(scopes ...string)
}
// CacheKey makes full key from primary key ans scopes
func CacheKey(key string, scopes ...string) string {
// Key makes full key from primary key ans scopes
func Key(key string, scopes ...string) string {
return strings.Join(scopes, "$$") + "@@" + key
}
func parseKey(fullKey string) (key string, scopes []string, err error) {
@@ -101,8 +102,11 @@ func (lc *loadingCache) Flush(scopes ...string) {
if len(scopes) == 0 {
lc.bytesCache.Flush()
go lc.postFlushFn()
return
}
// check if fullKey has matching scopes
inScope := func(fullKey string) bool {
for _, s := range scopes {
_, keyScopes, err := parseKey(fullKey)
@@ -118,18 +122,18 @@ func (lc *loadingCache) Flush(scopes ...string) {
return false
}
if len(scopes) > 0 {
matchedKeys := []string{}
lc.withLock(func() {
for k := range lc.activeKeys {
if inScope(k) {
matchedKeys = append(matchedKeys, k)
}
// all matchedKeys should be collected first
// we can't delete it from locked section, it will lock on eviction callback
matchedKeys := []string{}
lc.withLock(func() {
for k := range lc.activeKeys {
if inScope(k) {
matchedKeys = append(matchedKeys, k)
}
})
for _, mkey := range matchedKeys {
lc.bytesCache.Delete(mkey)
}
})
for _, mkey := range matchedKeys {
lc.bytesCache.Delete(mkey)
}
if lc.postFlushFn != nil {
@@ -188,8 +192,8 @@ func PostFlushFn(postFlushFn func()) CacheOption {
// admins will have different keys in order to prevent leak of admin-only data to regular users
func URLKey(r *http.Request) string {
adminPrefix := "admin!!"
key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
if user, err := GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
key = adminPrefix + key
}
return key
app/rest/cache_test.go → app/rest/cache/cache_test.go
Vendored
+8 -7
View File
@@ -1,4 +1,4 @@
package rest
package cache
import (
"fmt"
@@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/app/rest"
"github.com/umputun/remark/app/store"
)
@@ -139,7 +140,7 @@ func TestLoadingCache_URLKey(t *testing.T) {
assert.Equal(t, "http://blah/123?key=v&k2=v2", key)
user := store.User{Admin: true}
r = SetUserInfo(r, user)
r = rest.SetUserInfo(r, user)
key = URLKey(r)
assert.Equal(t, "admin!!http://blah/123?key=v&k2=v2", key)
}
@@ -175,25 +176,25 @@ func TestLoadingCache_Parallel(t *testing.T) {
func TestLoadingCache_Scopes(t *testing.T) {
lc := NewLoadingCache(CleanupInterval(time.Second))
res, err := lc.Get(CacheKey("key", "s1", "s2"), time.Minute, func() ([]byte, error) {
res, err := lc.Get(Key("key", "s1", "s2"), time.Minute, func() ([]byte, error) {
return []byte("value"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value", string(res))
res, err = lc.Get(CacheKey("key2", "s2"), time.Minute, func() ([]byte, error) {
res, err = lc.Get(Key("key2", "s2"), time.Minute, func() ([]byte, error) {
return []byte("value2"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value2", string(res))
lc.Flush("s1")
lc.Get(CacheKey("key2", "s2"), time.Minute, func() ([]byte, error) {
lc.Get(Key("key2", "s2"), time.Minute, func() ([]byte, error) {
assert.Fail(t, "should stay")
return nil, nil
})
res, err = lc.Get(CacheKey("key", "s1", "s2"), time.Minute, func() ([]byte, error) {
res, err = lc.Get(Key("key", "s1", "s2"), time.Minute, func() ([]byte, error) {
return []byte("value-upd"), nil
})
assert.Equal(t, "value-upd", string(res), "was deleted, update")
@@ -211,7 +212,7 @@ func TestLoadingCache_Keys(t *testing.T) {
}
for n, tt := range tbl {
full := CacheKey(tt.key, tt.scopes...)
full := Key(tt.key, tt.scopes...)
assert.Equal(t, tt.full, full, "making key, #%d", n)
k, s, e := parseKey(full)