diff --git a/backend/Gopkg.lock b/backend/Gopkg.lock index f9cd26da..995c89c0 100644 --- a/backend/Gopkg.lock +++ b/backend/Gopkg.lock @@ -130,6 +130,14 @@ revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b" version = "v1.0.0" +[[projects]] + branch = "master" + digest = "1:c6b263f17e06fcc612b40d1a4ca6d29588ae536170810bddd78227b5ea21f1f1" + name = "github.com/go-pkgz/rest" + packages = ["cache"] + pruneopts = "UT" + revision = "88a256cf379018b68f9ef9a35c79fd336ae503fe" + [[projects]] digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91" name = "github.com/golang/protobuf" @@ -376,6 +384,7 @@ "github.com/go-chi/render", "github.com/go-pkgz/mongo", "github.com/go-pkgz/repeater", + "github.com/go-pkgz/rest/cache", "github.com/google/uuid", "github.com/gorilla/feeds", "github.com/hashicorp/go-multierror", diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 77f980f7..ab7abae0 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -12,15 +12,15 @@ import ( "syscall" "time" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" "github.com/go-pkgz/mongo" + "github.com/go-pkgz/rest/cache" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/notify" "github.com/umputun/remark/backend/app/rest/api" "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" @@ -431,14 +431,14 @@ func (s *ServerCommand) makeCache() (cache.LoadingCache, error) { case "mem": return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), cache.MaxKeys(s.Cache.Max.Items)) - case "mongo": - mgServer, err := s.makeMongo() - if err != nil { - return nil, errors.Wrap(err, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache") - return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), - cache.MaxKeys(s.Cache.Max.Items)) + // case "mongo": + // mgServer, err := s.makeMongo() + // if err != nil { + // return nil, errors.Wrap(err, "failed to create mongo server") + // } + // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache") + // return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), + // cache.MaxKeys(s.Cache.Max.Items)) case "none": return &cache.Nop{}, nil } diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 3b5f67c1..82284d54 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -15,7 +15,7 @@ import ( "github.com/globalsign/mgo" "github.com/go-pkgz/mongo" - "github.com/jessevdk/go-flags" + flags "github.com/jessevdk/go-flags" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -91,7 +91,7 @@ func TestServerApp_WithMongo(t *testing.T) { // prepare options p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--dev-passwd=password", "--cache.type=mongo", "--store.type=mongo", + _, err := p.ParseArgs([]string{"--dev-passwd=password", "--cache.type=none", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"}) require.Nil(t, err) opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" diff --git a/backend/app/main.go b/backend/app/main.go index a6c39e0f..b266442b 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -77,19 +77,23 @@ func setupLog(dbg bool) { log.SetOutput(filter) } +// getDump reads runtime stack and returns as a string +func getDump() string { + maxSize := 5 * 1024 * 1024 + stacktrace := make([]byte, maxSize) + length := runtime.Stack(stacktrace, true) + if length > maxSize { + length = maxSize + } + return string(stacktrace[:length]) +} + func init() { // catch SIGQUIT and print stack traces sigChan := make(chan os.Signal) go func() { for range sigChan { - log.Print("[INFO] SIGQUIT detected") - maxSize := 5 * 1024 * 1024 - stacktrace := make([]byte, maxSize) - length := runtime.Stack(stacktrace, true) - if length > maxSize { - length = maxSize - } - fmt.Println(string(stacktrace[:length])) + log.Printf("[INFO] SIGQUIT detected, dump:\n%s", getDump()) } }() signal.Notify(sigChan, syscall.SIGQUIT) diff --git a/backend/app/main_test.go b/backend/app/main_test.go index 6d3dcc29..7981e1d6 100644 --- a/backend/app/main_test.go +++ b/backend/app/main_test.go @@ -2,8 +2,10 @@ package main import ( "io/ioutil" + "log" "net/http" "os" + "strings" "sync" "syscall" "testing" @@ -46,3 +48,11 @@ func TestMain(t *testing.T) { wg.Wait() } + +func TestGetDump(t *testing.T) { + dump := getDump() + assert.True(t, strings.Contains(dump, "goroutine")) + assert.True(t, strings.Contains(dump, "[running]")) + assert.True(t, strings.Contains(dump, "backend/app/main.go")) + log.Print("\n dump:" + dump) +} diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index cbb021a4..a904159c 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" @@ -24,20 +24,20 @@ func TestDisqus_Import(t *testing.T) { d := Disqus{DataStore: &dataStore} size, err := d.Import(strings.NewReader(xmlTestDisqus), "test") assert.Nil(t, err) - assert.Equal(t, 3, size) + assert.Equal(t, 4, size) last, err := dataStore.Last("test", 10) assert.Nil(t, err) - assert.Equal(t, 3, len(last), "3 comments imported") + assert.Equal(t, 4, len(last), "4 comments imported") - c := last[0] - assert.True(t, strings.HasPrefix(c.Text, "

Google App Engine")) - assert.Equal(t, "299986072", c.ID) + c := last[len(last)-1] // last reverses, get first one + assert.True(t, strings.HasPrefix(c.Text, "

The quick brown fox")) + assert.Equal(t, "299619020", c.ID) assert.Equal(t, "", c.ParentID) - assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator) - assert.Equal(t, "Dmitry Noname", c.User.Name) - assert.Equal(t, "disqus_8799342cdf328253e03313958ffc6a433659d7ff", c.User.ID) - assert.Equal(t, "7001968ea3f6c9013a9f0a3650f200c10c927638", c.User.IP) + assert.Equal(t, store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"}, c.Locator) + assert.Equal(t, "Alexander Blah", c.User.Name) + assert.Equal(t, "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", c.User.ID) + assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP) posts, err := dataStore.List("test", 0, 0) assert.Nil(t, err) @@ -56,7 +56,7 @@ func TestDisqus_Convert(t *testing.T) { for comment := range ch { res = append(res, comment) } - assert.Equal(t, 3, len(res), "3 comments total, 1 spam excluded") + assert.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded") exp0 := store.Comment{ ID: "299619020", @@ -102,6 +102,7 @@ var xmlTestDisqus = ` false false + http://www.radio-t.com/p/2011/03/05/podcast-229/ radiot @@ -121,6 +122,7 @@ var xmlTestDisqus = ` false + 3565798471341011339 @@ -175,6 +177,23 @@ var xmlTestDisqus = ` + + 12345678890 + This comment had no ID + 2011-08-31T22:49:43Z + radiot + false + false + + blah.noname@gmail.com + Blah Noname + false + 74b9e7568ef6860e93862c5d77590123 + + 189.89.89.139 + + + 6580890074280459219 some ugly spam @@ -190,5 +209,21 @@ var xmlTestDisqus = ` 189.89.89.139 + + + some bad comment + 2011-x09-30T22:48:43Z + false + 123 + + noname@gmail.com + Noname + true + google-2c5d77590123 + + 189.89.89.39 + + + ` diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 0ebcd09e..42367faf 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,11 +33,11 @@ func TestMigrator_ImportDisqus(t *testing.T) { Provider: "disqus", }) assert.Nil(t, err) - assert.Equal(t, 3, size) + assert.Equal(t, 4, size) last, err := dataStore.Last("test", 10) assert.Nil(t, err) - assert.Equal(t, 3, len(last), "3 comments imported") + assert.Equal(t, 4, len(last), "4 comments imported") } func TestMigrator_ImportWordPress(t *testing.T) { diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 1913d45a..5ee0dad6 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -9,10 +9,10 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" + "github.com/go-pkgz/rest/cache" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/service" diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 2c489023..2fdf0e4e 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -14,11 +14,11 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" + "github.com/go-pkgz/rest/cache" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" - "github.com/umputun/remark/backend/app/rest/cache" ) // Migrator rest with import and export controllers diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 031e8d04..dec91a4d 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -15,14 +15,14 @@ import ( "testing" "time" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" "github.com/go-chi/chi" + "github.com/go-pkgz/rest/cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 2e32e69b..4167a881 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -19,13 +19,13 @@ import ( "github.com/go-chi/chi/middleware" "github.com/go-chi/cors" "github.com/go-chi/render" + "github.com/go-pkgz/rest/cache" "github.com/pkg/errors" "github.com/rakyll/statik/fs" "github.com/umputun/remark/backend/app/notify" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/service" @@ -336,3 +336,14 @@ func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (fi } return filtered } + +// URLKey gets url from request to use it as cache key +// admins will have different keys in order to prevent leak of admin-only data to regular users +func URLKey(r *http.Request) string { + adminPrefix := "admin!!" + key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view + if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins + key = adminPrefix + key + } + return key +} diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 0331c911..a607f100 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -13,11 +13,11 @@ import ( jwt "github.com/dgrijalva/jwt-go" "github.com/go-chi/chi" "github.com/go-chi/render" + "github.com/go-pkgz/rest/cache" multierror "github.com/hashicorp/go-multierror" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/service" ) diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index b6582acd..45cfdc45 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -10,9 +10,9 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" + "github.com/go-pkgz/rest/cache" "github.com/umputun/remark/backend/app/rest" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/store" ) @@ -26,7 +26,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) { } log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format")) - key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL) + key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Find(locator, sort) if e != nil { @@ -87,7 +87,7 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) { func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} - key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL) + key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL) data, err := s.Cache.Get(key, func() ([]byte, error) { info, e := s.DataService.Info(locator, s.ReadOnlyAge) if e != nil { @@ -114,7 +114,7 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) { limit = 0 } - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(lastCommentsScope) + key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, limit) if e != nil { @@ -170,7 +170,7 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) { log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID) - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(userID, siteID) + key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(userID, siteID) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.User(siteID, userID, limit, 0) if e != nil { @@ -253,7 +253,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) { } // key could be long for multiple posts, make it sha1 - k := cache.URLKey(r) + strings.Join(posts, ",") + k := URLKey(r) + strings.Join(posts, ",") hasher := sha1.New() if _, err := hasher.Write([]byte(k)); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls") @@ -289,7 +289,7 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) { skip = v } - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID) + key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID) data, err := s.Cache.Get(key, func() ([]byte, error) { posts, e := s.DataService.List(siteID, limit, skip) if e != nil { diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index ebf563e5..f5600760 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -12,13 +12,13 @@ import ( "testing" "time" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" + "github.com/go-pkgz/rest/cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go index da0562cf..4c5b1256 100644 --- a/backend/app/rest/api/rss.go +++ b/backend/app/rest/api/rss.go @@ -7,11 +7,11 @@ import ( "time" "github.com/go-chi/chi" + "github.com/go-pkgz/rest/cache" "github.com/gorilla/feeds" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/rest" - "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/store" ) @@ -35,7 +35,7 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} log.Printf("[DEBUG] get rss for post %+v", locator) - key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL) + key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Find(locator, "-time") if e != nil { @@ -67,7 +67,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss for site %s", siteID) - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope) + key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, maxRssItems) if e != nil { @@ -100,7 +100,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID) - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope) + key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope) data, err := s.Cache.Get(key, func() (res []byte, e error) { comments, e := s.DataService.Last(siteID, maxLastCommentsReply) if e != nil { diff --git a/backend/app/rest/cache/cache_test.go b/backend/app/rest/cache/cache_test.go deleted file mode 100644 index 8b0e0645..00000000 --- a/backend/app/rest/cache/cache_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package cache - -import ( - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/umputun/remark/backend/app/rest" - "github.com/umputun/remark/backend/app/store" -) - -func TestCache_Keys(t *testing.T) { - tbl := []struct { - key string - scopes []string - full string - }{ - {"key1", []string{"s1"}, "s1@@key1@@site"}, - {"key2", []string{"s11", "s2"}, "s11$$s2@@key2@@site"}, - {"key3", []string{}, "@@key3@@site"}, - } - - for n, tt := range tbl { - k := NewKey("site").ID(tt.key).Scopes(tt.scopes...) - full := k.Merge() - assert.Equal(t, tt.full, full, "making key, #%d", n) - - k, e := ParseKey(full) - assert.Nil(t, e) - assert.Equal(t, tt.scopes, k.scopes) - assert.Equal(t, tt.key, k.id) - } - - _, err := ParseKey("abc") - assert.Error(t, err) - _, err = ParseKey("") - assert.Error(t, err) -} - -func TestCache_URLKey(t *testing.T) { - r, err := http.NewRequest("GET", "http://blah/123", nil) - assert.Nil(t, err) - key := URLKey(r) - assert.Equal(t, "http://blah/123", key) - - r, err = http.NewRequest("GET", "http://blah/123?key=v&k2=v2", nil) - assert.Nil(t, err) - key = URLKey(r) - assert.Equal(t, "http://blah/123?key=v&k2=v2", key) - - user := store.User{Admin: true} - r = rest.SetUserInfo(r, user) - key = URLKey(r) - assert.Equal(t, "admin!!http://blah/123?key=v&k2=v2", key) -} diff --git a/backend/app/rest/cache/memory_test.go b/backend/app/rest/cache/memory_test.go deleted file mode 100644 index 35fe6b6a..00000000 --- a/backend/app/rest/cache/memory_test.go +++ /dev/null @@ -1,310 +0,0 @@ -package cache - -import ( - "fmt" - "math/rand" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMemoryCache_Get(t *testing.T) { - var postFnCall, coldCalls int32 - lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) })) - require.Nil(t, err) - res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte("result"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result", string(res)) - assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - - res, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte("result"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result", string(res)) - assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - - lc.Flush(Flusher("site")) - time.Sleep(100 * time.Millisecond) // let postFn to do its thing - assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall)) - - _, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - return nil, errors.New("err") - }) - assert.NotNil(t, err) -} - -func TestMemoryCache_MaxKeys(t *testing.T) { - var postFnCall, coldCalls int32 - lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }), - MaxKeys(5), MaxValSize(10)) - require.Nil(t, err) - - // put 5 keys to cache - for i := 0; i < 5; i++ { - res, e := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte(fmt.Sprintf("result-%d", i)), nil - }) - assert.Nil(t, e) - assert.Equal(t, fmt.Sprintf("result-%d", i), string(res)) - assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - } - - // check if really cached - res, err := lc.Get(NewKey("site").ID("key-3"), func() ([]byte, error) { - return []byte("result-blah"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-3", string(res), "should be cached") - - // try to cache after maxKeys reached - res, err = lc.Get(NewKey("site").ID("key-X"), func() ([]byte, error) { - return []byte("result-X"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-X", string(res)) - - assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len()) - - // put to cache and make sure it cached - res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - - res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Zzzz"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res), "got cached value") - assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len()) -} - -func TestMemoryCache_MaxValueSize(t *testing.T) { - lc, err := NewMemoryCache(MaxKeys(5), MaxValSize(10)) - require.Nil(t, err) - // put good size value to cache and make sure it cached - res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - - res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Zzzz"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res), "got cached value") - - // put too big value to cache and make sure it is not cached - res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { - return []byte("1234567890"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "1234567890", string(res)) - - res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { - return []byte("result-big"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-big", string(res), "got not cached value") -} - -func TestMemoryCache_MaxCacheSize(t *testing.T) { - lc, err := NewMemoryCache(MaxKeys(50), MaxCacheSize(20)) - require.Nil(t, err) - - // put good size value to cache and make sure it cached - res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - assert.Equal(t, int64(8), lc.(*memoryCache).currentSize) - - _, err = lc.Get(NewKey("site").ID("key-Z2"), func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, int64(16), lc.(*memoryCache).currentSize) - - // this will cause removal - _, err = lc.Get(NewKey("site").ID("key-Z3"), func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, int64(16), lc.(*memoryCache).currentSize) - - assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len()) -} - -func TestMemoryCache_MaxCacheSizeParallel(t *testing.T) { - lc, err := NewMemoryCache(MaxCacheSize(123), MaxKeys(10000)) - require.Nil(t, err) - - wg := sync.WaitGroup{} - for i := 0; i < 1000; i++ { - wg.Add(1) - i := i - go func() { - time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond) - defer wg.Done() - res, err := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) { - return []byte(fmt.Sprintf("result-%d", i)), nil - }) - require.Nil(t, err) - require.Equal(t, fmt.Sprintf("result-%d", i), string(res)) - size := atomic.LoadInt64(&lc.(*memoryCache).currentSize) - require.True(t, size < 200 && size >= 0, "unexpected size=%d", size) // won't be exactly 123 due parallel - }() - } - wg.Wait() - assert.True(t, lc.(*memoryCache).currentSize < 123 && lc.(*memoryCache).currentSize >= 0) - t.Log("size=", lc.(*memoryCache).currentSize) -} - -func TestMemoryCache_Parallel(t *testing.T) { - var coldCalls int32 - lc, err := NewMemoryCache() - require.Nil(t, err) - - res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(res)) - - wg := sync.WaitGroup{} - for i := 0; i < 1000; i++ { - wg.Add(1) - i := i - go func() { - defer wg.Done() - res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte(fmt.Sprintf("result-%d", i)), nil - }) - require.Nil(t, err) - require.Equal(t, "value", string(res)) - }() - } - wg.Wait() - assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls)) -} - -func TestMemoryCache_Scopes(t *testing.T) { - lc, err := NewMemoryCache() - require.Nil(t, err) - - res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(res)) - - res, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { - return []byte("value2"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value2", string(res)) - - assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len()) - lc.Flush(Flusher("site").Scopes("s1")) - assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) - - _, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { - assert.Fail(t, "should stay") - return nil, nil - }) - assert.Nil(t, err) - res, err = lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { - return []byte("value-upd"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value-upd", string(res), "was deleted, update") -} - -func TestMemoryCache_Flush(t *testing.T) { - lc, err := NewMemoryCache() - require.Nil(t, err) - - addToCache := func(id string, scopes ...string) { - res, err := lc.Get(NewKey("site").ID(id).Scopes(scopes...), func() ([]byte, error) { - return []byte("value" + id), nil - }) - require.Nil(t, err) - require.Equal(t, "value"+id, string(res)) - } - - init := func() { - lc.Flush(Flusher("site")) - addToCache("key1", "s1", "s2") - addToCache("key2", "s1", "s2", "s3") - addToCache("key3", "s1", "s2", "s3") - addToCache("key4", "s2", "s3") - addToCache("key5", "s2") - addToCache("key6") - addToCache("key7", "s4", "s3") - require.Equal(t, 7, lc.(*memoryCache).bytesCache.Len(), "cache init") - } - - tbl := []struct { - scopes []string - left int - msg string - }{ - {[]string{}, 0, "full flush, no scopes"}, - {[]string{"s0"}, 7, "flush wrong scope"}, - {[]string{"s1"}, 4, "flush s1 scope"}, - {[]string{"s2", "s1"}, 2, "flush s2+s1 scope"}, - {[]string{"s1", "s2"}, 2, "flush s1+s2 scope"}, - {[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"}, - {[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"}, - {[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"}, - } - - for i, tt := range tbl { - init() - lc.Flush(Flusher("site").Scopes(tt.scopes...)) - assert.Equal(t, tt.left, lc.(*memoryCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i) - } -} - -func TestMemoryCache_FlushFailed(t *testing.T) { - lc, err := NewMemoryCache() - require.Nil(t, err) - val, err := lc.Get(NewKey("site").ID("invalid-composite"), func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(val)) - assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) - - lc.Flush(Flusher("site").Scopes("invalid-composite")) - assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) -} - -func TestMemoryCache_BadOptions(t *testing.T) { - _, err := NewMemoryCache(MaxCacheSize(-1)) - assert.EqualError(t, err, "failed to set cache option: negative size or MaxCacheSize, -1") - - _, err = NewMemoryCache(MaxKeys(-1)) - assert.EqualError(t, err, "failed to set cache option: negative size for MaxKeys, -1") - - _, err = NewMemoryCache(MaxValSize(-1)) - assert.EqualError(t, err, "failed to set cache option: negative size for MaxValSize, -1") -} diff --git a/backend/app/rest/cache/mongo.go b/backend/app/rest/cache/mongo.go deleted file mode 100644 index b0e23b5a..00000000 --- a/backend/app/rest/cache/mongo.go +++ /dev/null @@ -1,187 +0,0 @@ -package cache - -import ( - "log" - "time" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/go-pkgz/mongo" - "github.com/go-pkgz/repeater" - multierror "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" -) - -type mongoCache struct { - connection *mongo.Connection - postFlushFn func() - maxKeys int - maxValueSize int - maxCacheSize int64 -} - -const cacheCollection = "cache" - -type mongoDoc struct { - SiteID string `bson:"site"` - Key string `bson:"key"` - Scopes []string `bson:"scopes,omitempty"` - Data []byte `bson:"data"` -} - -// NewMongoCache makes mongoCache implementation -func NewMongoCache(connection *mongo.Connection, options ...Option) (LoadingCache, error) { - log.Printf("[INFO] make mongo cache with %s", connection) - res := &mongoCache{ - connection: connection, - postFlushFn: func() {}, - maxKeys: 1000, - maxValueSize: 0, - } - for _, opt := range options { - if err := opt(res); err != nil { - return nil, errors.Wrap(err, "failed to set cache option") - } - } - if err := res.prepare(); err != nil { - return nil, err - } - return res, nil -} - -// Get is loading cache method to get value by key or load via fn if not found -func (m *mongoCache) Get(key Key, fn func() ([]byte, error)) (data []byte, err error) { - - d := mongoDoc{} - - // repeat find from cache with small delay to avoid mgo random error - rep := repeater.NewDefault(5, 10*time.Millisecond) - mgErr := rep.Do(func() error { - return m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": key.siteID, "key": key.id}).One(&d) - }) - }, mgo.ErrNotFound) - if mgErr == nil { // cached result found - return d.Data, nil - } - - if data, err = fn(); err != nil { - return data, err - } - - if mgErr != mgo.ErrNotFound { // some other error in mgo query, don't try to update cache - log.Printf("[WARN] unexpected mgo error %+v", mgErr) - return data, err - } - - if !m.allowed(data) { - return data, nil - } - - d = mongoDoc{ - SiteID: key.siteID, - Key: key.id, - Data: data, - Scopes: key.scopes, - } - err = m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { - _, e := coll.Upsert(bson.M{"site": key.siteID, "key": key.id}, bson.M{"$set": d}) - return e - }) - if err != nil { - return nil, errors.Wrapf(err, "can't set cached value for %+v", key) - } - - if m.maxKeys > 0 { - err = m.cleanup(key.siteID) - } - - return data, errors.Wrap(err, "failed to cleanup cached records") -} - -func (m *mongoCache) cleanup(siteID string) (err error) { - ids := []struct { - ID bson.ObjectId `bson:"_id"` - }{} - - err = m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { - n, countErr := coll.Find(bson.M{"site": siteID}).Count() - if countErr != nil { - return countErr - } - if countErr == nil && n > m.maxKeys { - if findErr := coll.Find(bson.M{"site": siteID}).Sort("+id").Limit(n - m.maxKeys).All(&ids); findErr == nil { - bsonIDs := []bson.ObjectId{} - for _, id := range ids { - bsonIDs = append(bsonIDs, id.ID) - } - _, removalErr := coll.RemoveAll(bson.M{"_id": bson.M{"$in": bsonIDs}}) - return removalErr - } - } - return nil - }) - return err -} - -// Flush clears cache and calls postFlushFn async -func (m *mongoCache) Flush(req FlusherRequest) { - err := m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { - q := bson.M{"site": req.siteID} - if len(req.scopes) > 0 { - q["scopes"] = bson.M{"$in": req.scopes} - } - _, e := coll.RemoveAll(q) - return e - }) - - if err == nil && m.postFlushFn != nil { - m.postFlushFn() - } -} - -// prepare collections with all indexes -func (m *mongoCache) prepare() error { - errs := new(multierror.Error) - return m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "key")) - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "scopes")) - return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", cacheCollection) - }) -} - -func (m *mongoCache) allowed(data []byte) bool { - if m.maxValueSize > 0 && len(data) >= m.maxValueSize { - return false - } - return true -} - -func (m *mongoCache) setMaxValSize(max int) error { - m.maxValueSize = max - if max <= 0 { - return errors.Errorf("negative size for MaxValSize, %d", max) - } - return nil -} - -func (m *mongoCache) setMaxKeys(max int) error { - m.maxKeys = max - if max <= 0 { - return errors.Errorf("negative size for MaxKeys, %d", max) - } - return nil -} - -func (m *mongoCache) setMaxCacheSize(max int64) error { - m.maxCacheSize = max - if max <= 0 { - return errors.Errorf("negative size or MaxCacheSize, %d", max) - } - return nil -} - -func (m *mongoCache) setPostFlushFn(postFlushFn func()) error { - m.postFlushFn = postFlushFn - return nil -} diff --git a/backend/app/rest/cache/mongo_test.go b/backend/app/rest/cache/mongo_test.go deleted file mode 100644 index aa478020..00000000 --- a/backend/app/rest/cache/mongo_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package cache - -import ( - "fmt" - "log" - "os" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/go-pkgz/mongo" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMongoCache_Get(t *testing.T) { - conn, err := mongo.MakeTestConnection(t) - assert.NoError(t, err) - defer mongo.RemoveTestCollections(t, conn, "cache") - - var postFnCall, coldCalls int32 - lc, err := NewMongoCache(conn, PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) })) - require.Nil(t, err) - res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte("result"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result", string(res)) - assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - - res, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte("result"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result", string(res)) - assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - - lc.Flush(Flusher("site")) - time.Sleep(100 * time.Millisecond) // let postFn to do its thing - assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall)) - - _, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { - return nil, errors.New("err") - }) - assert.NotNil(t, err) -} - -func TestMongoCache_MaxKeys(t *testing.T) { - var postFnCall, coldCalls int32 - conn, err := mongo.MakeTestConnection(t) - assert.NoError(t, err) - defer mongo.RemoveTestCollections(t, conn, "cache") - - lc, err := NewMongoCache(conn, PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }), - MaxKeys(5), MaxValSize(10)) - require.Nil(t, err) - - // put 5 keys to cache - for i := 0; i < 5; i++ { - res, e := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte(fmt.Sprintf("result-%d", i)), nil - }) - assert.Nil(t, e) - assert.Equal(t, fmt.Sprintf("result-%d", i), string(res)) - assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls)) - assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - } - - // check if really cached - res, err := lc.Get(NewKey("site").ID("key-3"), func() ([]byte, error) { - return []byte("result-blah"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-3", string(res), "should be cached") - - // try to cache after maxKeys reached - res, err = lc.Get(NewKey("site").ID("key-X"), func() ([]byte, error) { - return []byte("result-X"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-X", string(res)) - - conn.WithCustomCollection("cache", func(coll *mgo.Collection) error { - n, e := coll.Find(bson.M{"site": "site"}).Count() - require.NoError(t, e) - require.Equal(t, 5, n) - r := mongoDoc{} - require.NoError(t, coll.Find(bson.M{"site": "site"}).Sort("+_id").One(&r)) - assert.Equal(t, "key-1", r.Key) - return nil - }) - - // put to cache and make sure it cached - res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - - res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Zzzz"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res), "got cached value") - - conn.WithCustomCollection("cache", func(coll *mgo.Collection) error { - n, e := coll.Find(bson.M{"site": "site"}).Count() - require.NoError(t, e) - require.Equal(t, 5, n) - r := mongoDoc{} - require.NoError(t, coll.Find(bson.M{"site": "site"}).Sort("+_id").One(&r)) - assert.Equal(t, "key-2", r.Key) - return nil - }) -} - -func TestMongoCache_MaxValueSize(t *testing.T) { - conn, err := mongo.MakeTestConnection(t) - assert.NoError(t, err) - defer mongo.RemoveTestCollections(t, conn, "cache") - lc, err := NewMongoCache(conn, MaxKeys(5), MaxValSize(10)) - require.Nil(t, err) - - // put good size value to cache and make sure it cached - res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Z"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res)) - - res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { - return []byte("result-Zzzz"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-Z", string(res), "got cached value") - - // put too big value to cache and make sure it is not cached - res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { - return []byte("1234567890"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "1234567890", string(res)) - - res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { - return []byte("result-big"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "result-big", string(res), "got not cached value") -} -func TestMongoCache_Parallel(t *testing.T) { - var coldCalls int32 - conn, err := mongo.MakeTestConnection(t) - assert.NoError(t, err) - defer mongo.RemoveTestCollections(t, conn, "cache") - lc, err := NewMongoCache(conn) - require.Nil(t, err) - - res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(res)) - wg := sync.WaitGroup{} - for i := 0; i < 100; i++ { - wg.Add(1) - i := i - go func() { - defer wg.Done() - r, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { - atomic.AddInt32(&coldCalls, 1) - return []byte(fmt.Sprintf("result-%d", i)), nil - }) - require.Nil(t, err) - v := string(r) - assert.Equal(t, "value", v, "th=%d", i) - }() - } - wg.Wait() - assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls)) -} - -func TestMongoCache_Flush(t *testing.T) { - conn, err := mongo.MakeTestConnection(t) - assert.NoError(t, err) - defer mongo.RemoveTestCollections(t, conn, "cache") - lc, err := NewMongoCache(conn) - require.Nil(t, err) - - addToCache := func(id string, scopes ...string) { - res, err := lc.Get(NewKey("site").ID(id).Scopes(scopes...), func() ([]byte, error) { - return []byte("value" + id), nil - }) - require.Nil(t, err) - require.Equal(t, "value"+id, string(res)) - } - - init := func() { - lc.Flush(Flusher("site")) - addToCache("key1", "s1", "s2") - addToCache("key2", "s1", "s2", "s3") - addToCache("key3", "s1", "s2", "s3") - addToCache("key4", "s2", "s3") - addToCache("key5", "s2") - addToCache("key6") - addToCache("key7", "s4", "s3") - require.Equal(t, 7, mongoCacheSize(t, conn), "cache init") - } - - tbl := []struct { - scopes []string - left int - msg string - }{ - {[]string{}, 0, "full flush, no scopes"}, - {[]string{"s0"}, 7, "flush wrong scope"}, - {[]string{"s1"}, 4, "flush s1 scope"}, - {[]string{"s2", "s1"}, 2, "flush s2+s1 scope"}, - {[]string{"s1", "s2"}, 2, "flush s1+s2 scope"}, - {[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"}, - {[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"}, - {[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"}, - } - - for i, tt := range tbl { - init() - lc.Flush(Flusher("site").Scopes(tt.scopes...)) - assert.Equal(t, tt.left, mongoCacheSize(t, conn), "keys size, %s #%d", tt.msg, i) - } -} - -func TestMongoCache_Scopes(t *testing.T) { - conn, err := mongo.MakeTestConnection(t) - assert.NoError(t, err) - defer mongo.RemoveTestCollections(t, conn, "cache") - lc, err := NewMongoCache(conn) - require.Nil(t, err) - - res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { - return []byte("value"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value", string(res)) - - res, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { - return []byte("value2"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value2", string(res)) - - assert.Equal(t, 2, mongoCacheSize(t, conn)) - lc.Flush(Flusher("site").Scopes("s1")) - assert.Equal(t, 1, mongoCacheSize(t, conn)) - - _, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { - assert.Fail(t, "should stay") - return nil, nil - }) - assert.Nil(t, err) - res, err = lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { - return []byte("value-upd"), nil - }) - assert.Nil(t, err) - assert.Equal(t, "value-upd", string(res), "was deleted, update") -} - -func BenchmarkMongoCache(b *testing.B) { - log.Print("[DEBUG] connect to mongo test instance") - srv, err := mongo.NewServerWithURL(os.Getenv("MONGO_TEST"), 10*time.Second) - assert.Nil(b, err, "failed to dial") - collName := fmt.Sprintf("test_%d", time.Now().Nanosecond()) - conn := mongo.NewConnection(srv, "test", collName) - - data := "" - for i := 0; i < 1000; i++ { - data += "x" - } - lc, err := NewMongoCache(conn) - require.Nil(b, err) - res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { - return []byte(data), nil - }) - require.Nil(b, err) - require.True(b, strings.HasPrefix(string(res), "xxxx"), string(res)) - - key := NewKey("site").ID("key").Scopes("s1", "s2") - b.ResetTimer() - for i := 0; i < b.N; i++ { - lc.Get(key, func() ([]byte, error) { - b.Fail() - return nil, nil - }) - } -} - -func mongoCacheSize(t *testing.T, conn *mongo.Connection) (count int) { - conn.WithCustomCollection("cache", func(coll *mgo.Collection) (e error) { - count, e = coll.Find(bson.M{"site": "site"}).Count() - require.NoError(t, e) - return e - }) - return count -} diff --git a/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/davecgh/go-spew/LICENSE b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/davecgh/go-spew/LICENSE new file mode 100644 index 00000000..c8364161 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/davecgh/go-spew/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/go-chi/chi/LICENSE b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/go-chi/chi/LICENSE new file mode 100644 index 00000000..d99f02ff --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/go-chi/chi/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2015-present Peter Kieltyka (https://github.com/pkieltyka), Google Inc. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/go-chi/render/LICENSE b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/go-chi/render/LICENSE new file mode 100644 index 00000000..4344db78 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/go-chi/render/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2016-Present https://github.com/go-chi authors + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/pkg/errors/LICENSE b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/pkg/errors/LICENSE new file mode 100644 index 00000000..835ba3e7 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/pkg/errors/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2015, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/pmezard/go-difflib/LICENSE b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/pmezard/go-difflib/LICENSE new file mode 100644 index 00000000..c67dad61 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/pmezard/go-difflib/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013, Patrick Mezard +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + The names of its contributors may not be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/stretchr/testify/LICENSE b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/stretchr/testify/LICENSE new file mode 100644 index 00000000..473b670a --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/.vendor/github.com/stretchr/testify/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell + +Please consider promoting this project if you find it useful. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT +OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/rest/LICENSE b/backend/vendor/github.com/go-pkgz/rest/LICENSE new file mode 100644 index 00000000..ca125214 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Umputun + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/rest/cache/README.md b/backend/vendor/github.com/go-pkgz/rest/cache/README.md new file mode 100644 index 00000000..8b0cef8f --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/rest/cache/README.md @@ -0,0 +1,43 @@ + +## Cache wrapper for web applications. + +The primary goal is to simplify caching of responses. + +Adds guava-style loading cache and support of scopes for partial flushes. +Provides in-memory `NewMemoryCache` on top of [hashicorp/golang-lru]("https://github.com/hashicorp/golang-lru") and +defines basic interface for other implementations. + +In addition to `Get` and `Flush` methods, memory cache also support limits for a single value size, number of keys and total memory utilization. `PostFlushFn` adds ability to call a function on flush completion. + +## Install and update + +`go get -u github.com/go-pkgz/rest/cache` + +## Technical details + +- Cache keeps data in a simple key:value format. +- Key is a type, created with `Key(site_id)` where `site_id` represents independent bucket in the cache. For simple cases can be set to an empty string. +- Particular key set by `Key.ID(string)` +- Key may contain optional scopes (list of string). They not affect retrieval and used for partial (scoped) invalidation only. +- Cache is safe for concurrent use. +- Value is []byte. +- `Get` method returns from the cache if the key already in. Overwise executes passed function and saves results. +- Special fake implementation `cache.Nop` satisfies `LoadingCache` interface and can be used to disable any caching + +## Usage + +```golang + // create in-memory cache with max keys=50, total (max) size=2000 and max cached size of a record = 200 + lc, err := cache.NewMemoryCache(cache.MaxKeys(50), cacheMaxCacheSize(2000), cache.MaxValSize(200)) + if err != nil { + panic(err) + } + ... + + // load cached value for key1. Call func if not cached yet or evicted + res, err = lc.Get(cache.NewKey("site1").ID("key1").Scopes("scope1"), func() ([]byte, error) { + return []byte("1234567890"), nil + }) + + lc.Flush("scope1") // invalidate cache for scope1 +``` diff --git a/backend/app/rest/cache/cache.go b/backend/vendor/github.com/go-pkgz/rest/cache/cache.go similarity index 79% rename from backend/app/rest/cache/cache.go rename to backend/vendor/github.com/go-pkgz/rest/cache/cache.go index b0c4d62c..2270cd8a 100644 --- a/backend/app/rest/cache/cache.go +++ b/backend/vendor/github.com/go-pkgz/rest/cache/cache.go @@ -1,12 +1,9 @@ package cache import ( - "net/http" "strings" "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/rest" ) // LoadingCache defines interface for caching @@ -89,17 +86,6 @@ func (f FlusherRequest) Scopes(scopes ...string) FlusherRequest { return f } -// URLKey gets url from request to use it as cache key -// admins will have different keys in order to prevent leak of admin-only data to regular users -func URLKey(r *http.Request) string { - adminPrefix := "admin!!" - key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view - if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins - key = adminPrefix + key - } - return key -} - // Nop does nothing for caching, passing fn call only type Nop struct{} diff --git a/backend/app/rest/cache/memory.go b/backend/vendor/github.com/go-pkgz/rest/cache/memory.go similarity index 100% rename from backend/app/rest/cache/memory.go rename to backend/vendor/github.com/go-pkgz/rest/cache/memory.go diff --git a/backend/app/rest/cache/options.go b/backend/vendor/github.com/go-pkgz/rest/cache/options.go similarity index 100% rename from backend/app/rest/cache/options.go rename to backend/vendor/github.com/go-pkgz/rest/cache/options.go