From 6abb2727c4956a557889dcd0c2375a4b42150d1a Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 11 Nov 2019 18:50:09 -0600 Subject: [PATCH] switch to lcw cache --- backend/app/cmd/server.go | 18 ++++++++++++++---- backend/app/rest/api/admin.go | 4 ++-- backend/app/rest/api/admin_test.go | 5 +++-- backend/app/rest/api/migrator.go | 4 ++-- backend/app/rest/api/rest.go | 10 ++++++++-- backend/app/rest/api/rest_private.go | 4 ++-- backend/app/rest/api/rest_public.go | 4 ++-- backend/app/rest/api/rest_public_test.go | 2 +- backend/app/rest/api/rest_test.go | 7 ++++--- backend/app/rest/api/rss.go | 4 ++-- 10 files changed, 40 insertions(+), 22 deletions(-) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 6fac7b89..04b465a4 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -26,7 +26,7 @@ import ( "github.com/go-pkgz/auth/provider" "github.com/go-pkgz/auth/provider/sender" "github.com/go-pkgz/auth/token" - "github.com/go-pkgz/rest/cache" + cache "github.com/go-pkgz/lcw" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/notify" @@ -203,6 +203,12 @@ type RPCGroup struct { AuthPassword string `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"` } +// LoadingCache defines interface for caching +type LoadingCache interface { + Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return + Flush(req cache.FlusherRequest) // evict matched records +} + // serverApp holds all active objects type serverApp struct { *ServerCommand @@ -544,14 +550,18 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) { } } -func (s *ServerCommand) makeCache() (cache.LoadingCache, error) { +func (s *ServerCommand) makeCache() (LoadingCache, error) { log.Printf("[INFO] make cache, type=%s", s.Cache.Type) switch s.Cache.Type { case "mem": - return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), + backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), cache.MaxKeys(s.Cache.Max.Items)) + if err != nil { + return nil, errors.Wrap(err, "cache backend initialization") + } + return cache.NewScache(backend), nil case "none": - return &cache.Nop{}, nil + return cache.NewScache(&cache.Nop{}), nil } return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type) } diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 0fc6c5e3..7c907c8b 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -9,9 +9,9 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" "github.com/go-pkgz/auth" + cache "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/store" @@ -20,7 +20,7 @@ import ( // admin provides router for all requests available for admin users only type admin struct { dataService adminStore - cache cache.LoadingCache + cache LoadingCache authenticator *auth.Service readOnlyAge int migrator *Migrator diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index d66cb3ac..66b02892 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -15,8 +15,8 @@ import ( jwt "github.com/dgrijalva/jwt-go" "github.com/go-pkgz/auth/token" + cache "github.com/go-pkgz/lcw" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -343,7 +343,8 @@ func TestAdmin_Block(t *testing.T) { assert.Equal(t, "test test #1", comments.Comments[2].Text, "comment not removed and not cleared") assert.False(t, comments.Comments[2].Deleted, "not deleted") - srv.pubRest.cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time + srv.pubRest.cache = cache.NewScache(cache.NewNopCache()) // TODO: with lru cache it won't be refreshed and invalidated for long + // time time.Sleep(50 * time.Millisecond) res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time") assert.Equal(t, 200, code) diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 3394ec53..a7d21ee6 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -12,9 +12,9 @@ import ( "time" "github.com/go-chi/render" + cache "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/migrator" @@ -23,7 +23,7 @@ import ( // Migrator rest with import and export controllers type Migrator struct { - Cache cache.LoadingCache + Cache LoadingCache NativeImporter migrator.Importer DisqusImporter migrator.Importer WordPressImporter migrator.Importer diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index ea6dea9f..20d622e9 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -17,9 +17,9 @@ import ( "github.com/go-chi/cors" "github.com/go-chi/render" "github.com/go-pkgz/auth" + "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/go-pkgz/rest/logger" "github.com/pkg/errors" "github.com/rakyll/statik/fs" @@ -38,7 +38,7 @@ type Rest struct { DataService *service.DataStore Authenticator *auth.Service - Cache cache.LoadingCache + Cache LoadingCache ImageProxy *proxy.Image CommentFormatter *store.CommentFormatter Migrator *Migrator @@ -69,6 +69,12 @@ type Rest struct { rssRest rss } +// LoadingCache defines interface for caching +type LoadingCache interface { + Get(key lcw.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return + Flush(req lcw.FlusherRequest) // evict matched records +} + const hardBodyLimit = 1024 * 64 // limit size of body const lastCommentsScope = "last" diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index c0aed6f4..f8945e9b 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -14,9 +14,9 @@ import ( "github.com/go-chi/render" "github.com/go-pkgz/auth" "github.com/go-pkgz/auth/token" + cache "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/hashicorp/go-multierror" "github.com/umputun/remark/backend/app/notify" @@ -28,7 +28,7 @@ import ( type private struct { dataService privStore - cache cache.LoadingCache + cache LoadingCache readOnlyAge int commentFormatter *store.CommentFormatter imageService *image.Service diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index e07464b5..c2da73b3 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -13,9 +13,9 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" + cache "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/rest" @@ -26,7 +26,7 @@ import ( type public struct { dataService pubStore - cache cache.LoadingCache + cache LoadingCache readOnlyAge int commentFormatter *store.CommentFormatter imageService *image.Service diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go index fae7c88e..6362639b 100644 --- a/backend/app/rest/api/rest_public_test.go +++ b/backend/app/rest/api/rest_public_test.go @@ -12,9 +12,9 @@ import ( "testing" "time" + cache "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 7297a857..812402f3 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -20,9 +20,9 @@ import ( "github.com/go-pkgz/auth" "github.com/go-pkgz/auth/avatar" "github.com/go-pkgz/auth/token" + cache "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" - "github.com/go-pkgz/rest/cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -289,8 +289,9 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "remark42"}) require.Nil(t, err) - memCache, err := cache.NewMemoryCache() - assert.NoError(t, err) + cacheBackend, err := cache.NewExpirableCache() + require.NoError(t, err) + memCache := cache.NewScache(cacheBackend) astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com") restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}}) diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go index 8dafd5ab..e435b338 100644 --- a/backend/app/rest/api/rss.go +++ b/backend/app/rest/api/rss.go @@ -5,8 +5,8 @@ import ( "net/http" "time" + cache "github.com/go-pkgz/lcw" log "github.com/go-pkgz/lgr" - "github.com/go-pkgz/rest/cache" "github.com/gorilla/feeds" "github.com/pkg/errors" @@ -16,7 +16,7 @@ import ( type rss struct { dataService rssStore - cache cache.LoadingCache + cache LoadingCache } type rssStore interface {