From 08be684d76f919a6dbb8b55085dbc91e53ddc776 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 25 May 2018 02:08:54 -0500 Subject: [PATCH] cache to separate package --- app/main.go | 6 +++--- app/rest/api/admin.go | 3 ++- app/rest/api/import.go | 3 ++- app/rest/api/rest.go | 15 +++++++------ app/rest/api/rss.go | 10 +++++---- app/rest/{ => cache}/cache.go | 34 +++++++++++++++++------------- app/rest/{ => cache}/cache_test.go | 15 +++++++------ 7 files changed, 48 insertions(+), 38 deletions(-) rename app/rest/{ => cache}/cache.go (84%) rename app/rest/{ => cache}/cache_test.go (93%) diff --git a/app/main.go b/app/main.go index db2d60ab..bcdcb07b 100644 --- a/app/main.go +++ b/app/main.go @@ -19,9 +19,9 @@ import ( "github.com/umputun/remark/app/store/service" "github.com/umputun/remark/app/migrator" - "github.com/umputun/remark/app/rest" "github.com/umputun/remark/app/rest/api" "github.com/umputun/remark/app/rest/auth" + "github.com/umputun/remark/app/rest/cache" "github.com/umputun/remark/app/rest/proxy" ) @@ -114,8 +114,8 @@ func New(opts Opts) (*Application, error) { MaxCommentSize: opts.MaxCommentSize, } - cache := rest.NewLoadingCache(rest.MaxValSize(opts.MaxCachedValue), rest.MaxKeys(opts.MaxCachedItems), - rest.PostFlushFn(postFlushFn(opts.Sites, opts.Port))) + cache := cache.NewLoadingCache(cache.MaxValSize(opts.MaxCachedValue), cache.MaxKeys(opts.MaxCachedItems), + cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port))) jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour) diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 1f55943d..e622c1a5 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -13,6 +13,7 @@ import ( "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest" + "github.com/umputun/remark/app/rest/cache" "github.com/umputun/remark/app/store" "github.com/umputun/remark/app/store/service" ) @@ -21,7 +22,7 @@ import ( type admin struct { dataService service.DataStore exporter migrator.Exporter - cache rest.LoadingCache + cache cache.LoadingCache defAvatarURL string } diff --git a/app/rest/api/import.go b/app/rest/api/import.go index 551b7ef2..b3ff12f9 100644 --- a/app/rest/api/import.go +++ b/app/rest/api/import.go @@ -17,12 +17,13 @@ import ( "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest" + "github.com/umputun/remark/app/rest/cache" ) // Import rest runs on unexposed port and available for local requests only type Import struct { Version string - Cache rest.LoadingCache + Cache cache.LoadingCache NativeImporter migrator.Importer DisqusImporter migrator.Importer SecretKey string diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 431cfc47..9cc4d76d 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -25,6 +25,7 @@ import ( "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest" "github.com/umputun/remark/app/rest/auth" + "github.com/umputun/remark/app/rest/cache" "github.com/umputun/remark/app/rest/proxy" "github.com/umputun/remark/app/store" "github.com/umputun/remark/app/store/service" @@ -36,7 +37,7 @@ type Rest struct { DataService service.DataStore Authenticator auth.Authenticator Exporter migrator.Exporter - Cache rest.LoadingCache + Cache cache.LoadingCache AvatarProxy *proxy.Avatar ImageProxy *proxy.Image WebRoot string @@ -308,7 +309,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) { } log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format")) - data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), locator.SiteID, locator.URL), 4*time.Hour, func() ([]byte, error) { + data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), 4*time.Hour, func() ([]byte, error) { comments, e := s.DataService.Find(locator, sort) if e != nil { return nil, e @@ -341,7 +342,7 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) { limit = 0 } - data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), "last", siteID), 4*time.Hour, func() ([]byte, error) { + data, err := s.Cache.Get(cache.Key(cache.URLKey(r), "last", siteID), 4*time.Hour, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, limit) if e != nil { return nil, e @@ -404,7 +405,7 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) { log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID) - data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), userID, siteID), 4*time.Hour, func() ([]byte, error) { + data, err := s.Cache.Get(cache.Key(cache.URLKey(r), userID, siteID), 4*time.Hour, func() ([]byte, error) { comments, count, e := s.DataService.User(siteID, userID, limit) if e != nil { return nil, e @@ -485,7 +486,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) { } // key could be long for multiple posts, make it sha1 - key := rest.URLKey(r) + strings.Join(posts, ",") + key := cache.URLKey(r) + strings.Join(posts, ",") hasher := sha1.New() if _, err := hasher.Write([]byte(key)); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls") @@ -493,7 +494,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) { } sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil)) - data, err := s.Cache.Get(rest.CacheKey(sha, siteID), 8*time.Hour, func() ([]byte, error) { + data, err := s.Cache.Get(cache.Key(sha, siteID), 8*time.Hour, func() ([]byte, error) { counts, e := s.DataService.Counts(siteID, posts) if e != nil { return nil, e @@ -521,7 +522,7 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) { skip = v } - data, err := s.Cache.Get(rest.CacheKey(rest.URLKey(r), siteID), 8*time.Hour, func() ([]byte, error) { + data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), 8*time.Hour, func() ([]byte, error) { posts, e := s.DataService.List(siteID, limit, skip) if e != nil { return nil, e diff --git a/app/rest/api/rss.go b/app/rest/api/rss.go index e7533b87..d5779776 100644 --- a/app/rest/api/rss.go +++ b/app/rest/api/rss.go @@ -11,6 +11,7 @@ import ( "github.com/gorilla/feeds" "github.com/umputun/remark/app/rest" + "github.com/umputun/remark/app/rest/cache" "github.com/umputun/remark/app/store" ) @@ -32,7 +33,7 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) { sort := "-time" log.Printf("[DEBUG] get rss for post %+v", locator) - data, err := s.Cache.Get(rest.URLKey(r), 4*time.Hour, func() ([]byte, error) { + data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), 4*time.Hour, func() ([]byte, error) { comments, e := s.DataService.Find(locator, sort) if e != nil { return nil, e @@ -61,10 +62,11 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) { // GET /rss/site?site=siteID func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) { - log.Printf("[DEBUG] get rss for site %s", r.URL.Query().Get("site")) + siteID := r.URL.Query().Get("site") + log.Printf("[DEBUG] get rss for site %s", siteID) - data, err := s.Cache.Get(rest.URLKey(r), 4*time.Hour, func() ([]byte, error) { - comments, e := s.DataService.Last(r.URL.Query().Get("site"), maxRssItems) + data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), 4*time.Hour, func() ([]byte, error) { + comments, e := s.DataService.Last(siteID, maxRssItems) if e != nil { return nil, e } diff --git a/app/rest/cache.go b/app/rest/cache/cache.go similarity index 84% rename from app/rest/cache.go rename to app/rest/cache/cache.go index c279fea0..ec00761c 100644 --- a/app/rest/cache.go +++ b/app/rest/cache/cache.go @@ -1,4 +1,4 @@ -package rest +package cache import ( "log" @@ -9,6 +9,7 @@ import ( "github.com/patrickmn/go-cache" "github.com/pkg/errors" + "github.com/umputun/remark/app/rest" ) // LoadingCache defines interface for caching @@ -17,8 +18,8 @@ type LoadingCache interface { Flush(scopes ...string) } -// CacheKey makes full key from primary key ans scopes -func CacheKey(key string, scopes ...string) string { +// Key makes full key from primary key ans scopes +func Key(key string, scopes ...string) string { return strings.Join(scopes, "$$") + "@@" + key } func parseKey(fullKey string) (key string, scopes []string, err error) { @@ -101,8 +102,11 @@ func (lc *loadingCache) Flush(scopes ...string) { if len(scopes) == 0 { lc.bytesCache.Flush() + go lc.postFlushFn() + return } + // check if fullKey has matching scopes inScope := func(fullKey string) bool { for _, s := range scopes { _, keyScopes, err := parseKey(fullKey) @@ -118,18 +122,18 @@ func (lc *loadingCache) Flush(scopes ...string) { return false } - if len(scopes) > 0 { - matchedKeys := []string{} - lc.withLock(func() { - for k := range lc.activeKeys { - if inScope(k) { - matchedKeys = append(matchedKeys, k) - } + // all matchedKeys should be collected first + // we can't delete it from locked section, it will lock on eviction callback + matchedKeys := []string{} + lc.withLock(func() { + for k := range lc.activeKeys { + if inScope(k) { + matchedKeys = append(matchedKeys, k) } - }) - for _, mkey := range matchedKeys { - lc.bytesCache.Delete(mkey) } + }) + for _, mkey := range matchedKeys { + lc.bytesCache.Delete(mkey) } if lc.postFlushFn != nil { @@ -188,8 +192,8 @@ func PostFlushFn(postFlushFn func()) CacheOption { // admins will have different keys in order to prevent leak of admin-only data to regular users func URLKey(r *http.Request) string { adminPrefix := "admin!!" - key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view - if user, err := GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins + key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view + if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins key = adminPrefix + key } return key diff --git a/app/rest/cache_test.go b/app/rest/cache/cache_test.go similarity index 93% rename from app/rest/cache_test.go rename to app/rest/cache/cache_test.go index 917af2ab..507d31fc 100644 --- a/app/rest/cache_test.go +++ b/app/rest/cache/cache_test.go @@ -1,4 +1,4 @@ -package rest +package cache import ( "fmt" @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/app/rest" "github.com/umputun/remark/app/store" ) @@ -139,7 +140,7 @@ func TestLoadingCache_URLKey(t *testing.T) { assert.Equal(t, "http://blah/123?key=v&k2=v2", key) user := store.User{Admin: true} - r = SetUserInfo(r, user) + r = rest.SetUserInfo(r, user) key = URLKey(r) assert.Equal(t, "admin!!http://blah/123?key=v&k2=v2", key) } @@ -175,25 +176,25 @@ func TestLoadingCache_Parallel(t *testing.T) { func TestLoadingCache_Scopes(t *testing.T) { lc := NewLoadingCache(CleanupInterval(time.Second)) - res, err := lc.Get(CacheKey("key", "s1", "s2"), time.Minute, func() ([]byte, error) { + res, err := lc.Get(Key("key", "s1", "s2"), time.Minute, func() ([]byte, error) { return []byte("value"), nil }) assert.Nil(t, err) assert.Equal(t, "value", string(res)) - res, err = lc.Get(CacheKey("key2", "s2"), time.Minute, func() ([]byte, error) { + res, err = lc.Get(Key("key2", "s2"), time.Minute, func() ([]byte, error) { return []byte("value2"), nil }) assert.Nil(t, err) assert.Equal(t, "value2", string(res)) lc.Flush("s1") - lc.Get(CacheKey("key2", "s2"), time.Minute, func() ([]byte, error) { + lc.Get(Key("key2", "s2"), time.Minute, func() ([]byte, error) { assert.Fail(t, "should stay") return nil, nil }) - res, err = lc.Get(CacheKey("key", "s1", "s2"), time.Minute, func() ([]byte, error) { + res, err = lc.Get(Key("key", "s1", "s2"), time.Minute, func() ([]byte, error) { return []byte("value-upd"), nil }) assert.Equal(t, "value-upd", string(res), "was deleted, update") @@ -211,7 +212,7 @@ func TestLoadingCache_Keys(t *testing.T) { } for n, tt := range tbl { - full := CacheKey(tt.key, tt.scopes...) + full := Key(tt.key, tt.scopes...) assert.Equal(t, tt.full, full, "making key, #%d", n) k, s, e := parseKey(full)