mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #202 from umputun/feature/avatar_migration

Browse Source 
feature/avatar_migration
This commit is contained in:
Umputun
2018-09-11 13:15:26 -05:00
committed by GitHub
parent a856fd9c7d a8b92a6039
commit f6708c12a1
34 changed files with 566 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile.artifacts
+8 -5
View File
@@ -30,13 +30,16 @@ WORKDIR /go/src/github.com/umputun/remark/backend
ADD backend /go/src/github.com/umputun/remark/backend
ADD README.md /go/src/github.com/umputun/remark/
ADD LICENSE /go/src/github.com/umputun/remark/
COPY --from=build-frontend /srv/web/public/ web
COPY --from=build-frontend /srv/web web
RUN \
RUN \
export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
go get -v github.com/rakyll/statik && \
statik --src=/go/src/github.com/umputun/remark/backend/web --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go
statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \
ls -la /go/src/github.com/umputun/remark/backend/web/
# if DRONE presented use DRONE_* git env to make version
RUN \
backend/app/cmd/avatar.go
+84
View File
@@ -0,0 +1,84 @@
package cmd
import (
"log"
"time"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store/avatar"
)
// AvatarCommand set of flags and command for avatar migration
// it converts all avatarts from src.type to dst.type
type AvatarCommand struct {
AvatarSrc AvatarGroup `group:"src" namespace:"src"`
AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
migrator AvatarMigrator
CommonOpts
}
// AvatarMigrator defines interface for migration
type AvatarMigrator interface {
Migrate(avatar.Store, avatar.Store) (int, error)
}
type avatarMigrator struct{}
func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
return avatar.Migrate(dst, src)
}
// Execute runs with AvatarCommand parameters, entry point for "avatar" command
func (ac *AvatarCommand) Execute(args []string) error {
log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
src, err := ac.makeAvatarStore(ac.AvatarSrc)
if err != nil {
return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type)
}
dst, err := ac.makeAvatarStore(ac.AvatarDst)
if err != nil {
return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type)
}
if ac.migrator == nil {
ac.migrator = avatarMigrator{}
}
count, err := ac.migrator.Migrate(dst, src)
if err != nil {
return err
}
log.Printf("[INFO] completed, migrated avatars = %d", count)
return nil
}
func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
switch gr.Type {
case "fs":
if err := makeDirs(gr.FS.Path); err != nil {
return nil, err
}
return avatar.NewLocalFS(gr.FS.Path, gr.RszLmt), nil
case "mongo":
mgServer, err := ac.makeMongo()
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
return avatar.NewGridFS(conn, gr.RszLmt), nil
}
return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
}
func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
if ac.Mongo.URL == "" {
return nil, errors.New("no mongo URL provided")
}
return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
}
backend/app/cmd/avatar_test.go
+53
View File
@@ -0,0 +1,53 @@
package cmd
import (
"errors"
"os"
"testing"
flags "github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/avatar"
)
func TestAvatar_Execute(t *testing.T) {
mongoURL := os.Getenv("MONGO_TEST")
if mongoURL == "" {
mongoURL = "mongodb://localhost:27017/test"
}
if mongoURL == "skip" {
t.Skip("skip mongo app test")
}
defer os.RemoveAll("/tmp/ava-test")
cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.Error(t, err, "failed blah")
}
type avatarMigratorMock struct {
called int
retError error
retCount int
}
func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) {
a.called++
return a.retCount, a.retError
}
backend/app/cmd/cmd.go
+29
View File
@@ -98,3 +98,32 @@ func responseError(resp *http.Response) error {
}
return errors.Errorf("error response %q, %s", resp.Status, body)
}
// mkdir -p for all dirs
func makeDirs(dirs ...string) error {
// exists returns whether the given file or directory exists or not
exists := func(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
for _, dir := range dirs {
ex, err := exists(dir)
if err != nil {
return errors.Wrapf(err, "can't check directory status for %s", dir)
}
if !ex {
if e := os.MkdirAll(dir, 0700); e != nil {
return errors.Wrapf(err, "can't make directory %s", dir)
}
}
}
return nil
}
backend/app/cmd/server.go
+17 -70
View File
@@ -24,7 +24,6 @@ import (
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -34,7 +33,6 @@ type ServerCommand struct {
Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"`
Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"`
Admin AdminGroup `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
@@ -105,11 +103,6 @@ type MongoGroup struct {
DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
}
// KeyGroup defines options group for key params
type KeyGroup struct {
Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"`
}
// AdminGroup defines options group for admin params
type AdminGroup struct {
Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
@@ -160,25 +153,20 @@ func (s *ServerCommand) Execute(args []string) error {
// doesn't start anything
func (s *ServerCommand) newServerApp() (*serverApp, error) {
if err := s.makeDirs(s.BackupLocation); err != nil {
if err := makeDirs(s.BackupLocation); err != nil {
return nil, err
}
if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") {
return nil, errors.Errorf("invalid remark42 url %s", s.RemarkURL)
}
log.Printf("[INFO] root url=%s", s.RemarkURL)
storeEngine, err := s.makeDataStore()
if err != nil {
return nil, errors.Wrap(err, "failed to make data store engine")
}
keyStore, err := s.makeKeyStore()
if err != nil {
return nil, errors.Wrap(err, "failed to make key store")
}
adminStore, err := s.makeAdminStore()
if err != nil {
return nil, errors.Wrap(err, "failed to make admin store")
@@ -187,7 +175,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
dataService := &service.DataStore{
Interface: storeEngine,
EditDuration: s.EditDuration,
KeyStore: keyStore,
AdminStore: adminStore,
MaxCommentSize: s.MaxCommentSize,
}
@@ -198,7 +185,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
}
// token TTL is 5 minutes, inactivity interval 7+ days by default
jwtService := auth.NewJWT(keyStore, strings.HasPrefix(s.RemarkURL, "https://"), s.Auth.TTL.JWT, s.Auth.TTL.Cookie)
jwtService := auth.NewJWT(adminStore, strings.HasPrefix(s.RemarkURL, "https://"), s.Auth.TTL.JWT, s.Auth.TTL.Cookie)
avatarStore, err := s.makeAvatarStore()
if err != nil {
@@ -218,7 +205,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
DisqusImporter: &migrator.Disqus{DataStore: dataService},
WordPressImporter: &migrator.WordPress{DataStore: dataService},
NativeExported: &migrator.Remark{DataStore: dataService},
KeyStore: keyStore,
KeyStore: adminStore,
}
authProviders := s.makeAuthProviders(jwtService, avatarProxy, dataService)
@@ -242,7 +229,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
Providers: authProviders,
DevPasswd: s.DevPasswd,
PermissionChecker: dataService,
KeyStore: keyStore,
},
Cache: loadingCache,
}
@@ -313,9 +299,11 @@ func (a *serverApp) activateBackup(ctx context.Context) {
// makeDataStore creates store for all sites
func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
log.Printf("[INFO] make data store, type=%s", s.Store.Type)
switch s.Store.Type {
case "bolt":
if err = s.makeDirs(s.Store.Bolt.Path); err != nil {
if err = makeDirs(s.Store.Bolt.Path); err != nil {
return nil, errors.Wrap(err, "failed to create bolt store")
}
sites := []engine.BoltSite{}
@@ -337,9 +325,11 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
}
func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
log.Printf("[INFO] make avatar store, type=%s", s.Avatar.Type)
switch s.Avatar.Type {
case "fs":
if err := s.makeDirs(s.Avatar.FS.Path); err != nil {
if err := makeDirs(s.Avatar.FS.Path); err != nil {
return nil, err
}
return avatar.NewLocalFS(s.Avatar.FS.Path, s.Avatar.RszLmt), nil
@@ -354,23 +344,8 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
}
func (s *ServerCommand) makeKeyStore() (keys.Store, error) {
switch s.Key.Type {
case "shared":
return keys.NewStaticStore(s.SharedSecret), nil
case "mongo":
mgServer, e := s.makeMongo()
if e != nil {
return nil, errors.Wrap(e, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
return keys.NewMongoStore(conn), nil
default:
return nil, errors.Errorf("unsupported key store type %s", s.Key.Type)
}
}
func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
switch s.Admin.Type {
case "shared":
@@ -379,7 +354,7 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
s.Admin.Shared.Email = "admin@" + u.Host
}
}
return admin.NewStaticStore(s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
case "mongo":
mgServer, e := s.makeMongo()
if e != nil {
@@ -388,11 +363,12 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
return admin.NewMongoStore(conn), nil
default:
return nil, errors.Errorf("unsupported admin store type %s", s.Key.Type)
return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
}
}
func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
switch s.Cache.Type {
case "mem":
return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
@@ -416,12 +392,12 @@ func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
}
func (s *ServerCommand) makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore) []auth.Provider {
func (s *ServerCommand) makeAuthProviders(jwt *auth.JWT, ap *proxy.Avatar, ds *service.DataStore) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
return auth.Params{
JwtService: jwtService,
AvatarProxy: avatarProxy,
JwtService: jwt,
AvatarProxy: ap,
RemarkURL: s.RemarkURL,
Cid: cid,
Csecret: secret,
@@ -451,32 +427,3 @@ func (s *ServerCommand) makeAuthProviders(jwtService *auth.JWT, avatarProxy *pro
}
return providers
}
// mkdir -p for all dirs
func (s *ServerCommand) makeDirs(dirs ...string) error {
// exists returns whether the given file or directory exists or not
exists := func(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
for _, dir := range dirs {
ex, err := exists(dir)
if err != nil {
return errors.Wrapf(err, "can't check directory status for %s", dir)
}
if !ex {
if e := os.MkdirAll(dir, 0700); e != nil {
return errors.Wrapf(err, "can't make directory %s", dir)
}
}
}
return nil
}
backend/app/cmd/server_test.go
+1 -2
View File
@@ -91,8 +91,7 @@ func TestServerApp_WithMongo(t *testing.T) {
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--dev-passwd=password", "--cache.type=mongo", "--store.type=mongo",
"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345",
"--key.type=mongo", "--admin.type=mongo"})
"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
require.Nil(t, err)
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.BackupLocation = "/tmp"
backend/app/main.go
+1
View File
@@ -17,6 +17,7 @@ type Opts struct {
ImportCmd cmd.ImportCommand `command:"import"`
BackupCmd cmd.BackupCommand `command:"backup"`
RestoreCmd cmd.RestoreCommand `command:"restore"`
AvatarCmd cmd.AvatarCommand `command:"avatar"`
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
backend/app/migrator/disqus_test.go
+2 -2
View File
@@ -9,8 +9,8 @@ import (
"github.com/coreos/bbolt"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
"github.com/stretchr/testify/assert"
@@ -20,7 +20,7 @@ func TestDisqus_Import(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
dataStore := service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}
dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
d := Disqus{DataStore: &dataStore}
size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
assert.Nil(t, err)
backend/app/migrator/migrator_test.go
+5 -5
View File
@@ -8,10 +8,10 @@ import (
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestMigrator_ImportDisqus(t *testing.T) {
@@ -25,7 +25,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}
dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "/tmp/disqus-test.xml",
@@ -51,7 +51,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}
dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "/tmp/wordpress-test.xml",
@@ -80,7 +80,7 @@ func TestMigrator_ImportRemark(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
require.Nil(t, err, "create store")
dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}
dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
backend/app/migrator/remark_test.go
+5 -4
View File
@@ -10,9 +10,10 @@ import (
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -50,7 +51,7 @@ func TestRemark_Import(t *testing.T) {
os.Remove(testDb)
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
r := Remark{DataStore: &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}}
r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
size, err := r.Import(buf, "radio-t")
assert.Nil(t, err)
assert.Equal(t, 2, size)
@@ -78,7 +79,7 @@ func TestRemark_ImportManyWithError(t *testing.T) {
os.Remove(testDb)
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
r := Remark{DataStore: &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}}
r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
n, err := r.Import(buf, "radio-t")
assert.EqualError(t, err, "failed to save 2 comments")
assert.Equal(t, 1200, n)
@@ -94,7 +95,7 @@ func prep(t *testing.T) *service.DataStore {
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
b := &service.DataStore{Interface: boltStore, KeyStore: keys.NewStaticStore("12345")}
b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
comment := store.Comment{
ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
backend/app/migrator/wordpress_test.go
+2 -2
View File
@@ -8,9 +8,9 @@ import (
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -21,7 +21,7 @@ func TestWordPress_Import(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
assert.Nil(t, err, "create store")
dataStore := service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}
dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
wp := WordPress{DataStore: &dataStore}
size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
assert.Nil(t, err)
backend/app/rest/api/migrator.go
+6 -2
View File
@@ -14,7 +14,6 @@ import (
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/store/keys"
)
// Migrator rest with import and export controllers
@@ -24,7 +23,12 @@ type Migrator struct {
DisqusImporter migrator.Importer
WordPressImporter migrator.Importer
NativeExported migrator.Exporter
KeyStore keys.Store
KeyStore KeyStore
}
// KeyStore defines sub-interface for consumers needed just a key
type KeyStore interface {
Key(siteID string) (key string, err error)
}
func (m *Migrator) withRoutes(router chi.Router) chi.Router {
backend/app/rest/api/migrator_test.go
+6 -8
View File
@@ -22,7 +22,6 @@ import (
"github.com/umputun/remark/backend/app/store"
adminstore "github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -153,29 +152,28 @@ func TestMigrator_Export(t *testing.T) {
func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
ks := keys.NewStaticStore("123456")
dataStore := &service.DataStore{Interface: b, KeyStore: ks}
adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
svc = &Migrator{
DisqusImporter: &migrator.Disqus{DataStore: dataStore},
WordPressImporter: &migrator.WordPress{DataStore: dataStore},
NativeImporter: &migrator.Remark{DataStore: dataStore},
NativeExported: &migrator.Remark{DataStore: dataStore},
Cache: &cache.Nop{},
KeyStore: ks,
KeyStore: adminStore,
}
a := auth.Authenticator{
DevPasswd: "password",
Providers: nil,
AdminStore: adminstore.NewStaticStore([]string{"a1", "a2"}, "admin@remark-42.com"),
JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour),
KeyStore: ks,
AdminStore: adminStore,
JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour),
}
routes := svc.withRoutes(chi.NewRouter().With(a.Auth(true)).With(a.AdminOnly))
ts = httptest.NewServer(routes)
return svc, dataStore, ts
}
func cleanupImportSrv(m *Migrator, ts *httptest.Server) {
func cleanupImportSrv(_ *Migrator, ts *httptest.Server) {
ts.Close()
os.Remove(testDb)
}
backend/app/rest/api/rest_test.go
+3 -5
View File
@@ -23,7 +23,6 @@ import (
adminstore "github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -92,13 +91,12 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
adminStore := adminstore.NewStaticStore([]string{"a1", "a2"}, "admin@remark-42.com")
adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
dataStore := &service.DataStore{
Interface: b,
EditDuration: 5 * time.Minute,
MaxCommentSize: 4000,
KeyStore: keys.NewStaticStore("123456"),
AdminStore: adminStore,
}
srv = &Rest{
@@ -107,7 +105,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
DevPasswd: "password",
Providers: nil,
AdminStore: adminStore,
JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour),
JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour),
},
Cache: &cache.Nop{},
WebRoot: "/tmp",
@@ -122,7 +120,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
NativeImporter: &migrator.Remark{DataStore: dataStore},
NativeExported: &migrator.Remark{DataStore: dataStore},
Cache: &cache.Nop{},
KeyStore: keys.NewStaticStore("123456"),
KeyStore: adminStore,
},
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
backend/app/rest/auth/auth.go
+2 -6
View File
@@ -10,7 +10,6 @@ import (
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/keys"
)
// Authenticator is top level auth object providing middlewares
@@ -18,7 +17,6 @@ type Authenticator struct {
JWTService *JWT
Providers []Provider
AdminStore admin.Store
KeyStore keys.Store
DevPasswd string
PermissionChecker PermissionChecker
}
@@ -114,14 +112,14 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler {
}
func (a *Authenticator) checkSecretKey(r *http.Request) bool {
if a.KeyStore == nil {
if a.AdminStore == nil {
return false
}
siteID := r.URL.Query().Get("site")
secret := r.URL.Query().Get("secret")
skey, err := a.KeyStore.Get(siteID)
skey, err := a.AdminStore.Key(siteID)
if err != nil {
return false
}
@@ -172,10 +170,8 @@ func (a *Authenticator) basicDevUser(r *http.Request) bool {
return false
}
log.Printf("[DEBUG] dev user auth")
s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
if len(s) != 2 {
log.Printf("[WARN] dev user auth failed, incorrect auth header %s", r.Header.Get("Authorization"))
return false
}
backend/app/rest/auth/auth_test.go
+6 -6
View File
@@ -11,7 +11,7 @@ import (
"github.com/go-chi/chi"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/admin"
)
var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOnRydWV9LCJzdGF0ZSI6IjEyMzQ1NiIsImZyb20iOiJmcm9tIn0.6P_OwGf8CUJRtvNSlW20GmaMb5pFvCNemP94fHCqb5Q"
@@ -19,7 +19,7 @@ var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxO
var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZmxhZ3MiOnsiZGVsZXRlbWUiOnRydWV9fQ.SLh1QpFytWZqcT99VgcdAOtgFKhvpKCcZwqWTvAd63g"
func TestAuthJWTCookie(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour),
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour),
PermissionChecker: &mockUserPermissions{}}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
@@ -57,7 +57,7 @@ func TestAuthJWTCookie(t *testing.T) {
}
func TestAuthJWTHeader(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)}
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
@@ -84,7 +84,7 @@ func TestAuthJWTHeader(t *testing.T) {
}
func TestAuthJWtBlocked(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)}
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
@@ -104,7 +104,7 @@ func TestAuthJWtBlocked(t *testing.T) {
}
func TestAuthJWtFlags(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)}
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
@@ -213,7 +213,7 @@ func TestAdminRequired(t *testing.T) {
}
func TestAuthWithSecret(t *testing.T) {
a := Authenticator{DevPasswd: "123456", KeyStore: keys.NewStaticStore("secretkey")}
a := Authenticator{DevPasswd: "123456", AdminStore: admin.NewStaticKeyStore("secretkey")}
router := chi.NewRouter()
router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
backend/app/rest/auth/dev_provider_test.go
+2 -2
View File
@@ -13,12 +13,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/admin"
)
func TestDevProvider(t *testing.T) {
params := Params{RemarkURL: "http://127.0.0.1:8080", Cid: "cid", Csecret: "csecret",
JwtService: NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31),
JwtService: NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31),
PermissionChecker: &mockUserPermissions{admin: "dev_user"},
}
srv := DevAuthServer{Provider: NewDev(params), nonInteractive: true, username: "dev_user"}
backend/app/rest/auth/jwt.go
+9 -5
View File
@@ -8,13 +8,12 @@ import (
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/keys"
)
// JWT wraps jwt operations
// supports both header and cookie jwt
type JWT struct {
keyStore keys.Store
keyStore KeyStore
secureCookies bool
tokenDuration time.Duration
cookieDuration time.Duration
@@ -43,8 +42,13 @@ const jwtHeaderKey = "X-JWT"
const xsrfCookieName = "XSRF-TOKEN"
const xsrfHeaderKey = "X-XSRF-TOKEN"
// KeyStore defines sub-interface for consumers needed just a key
type KeyStore interface {
Key(siteID string) (key string, err error)
}
// NewJWT makes JWT service
func NewJWT(keyStore keys.Store, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT {
func NewJWT(keyStore KeyStore, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT {
res := JWT{
keyStore: keyStore,
secureCookies: secureCookies,
@@ -58,7 +62,7 @@ func NewJWT(keyStore keys.Store, secureCookies bool, tokenDuration time.Duration
func (j *JWT) Token(claims *CustomClaims) (string, error) {
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
secret, err := j.keyStore.Get(claims.SiteID)
secret, err := j.keyStore.Key(claims.SiteID)
if err != nil {
return "", errors.Wrap(err, "can't get secret")
}
@@ -96,7 +100,7 @@ func (j *JWT) Parse(tokenString string) (*CustomClaims, error) {
return nil, errors.Wrap(err, "failed to get siteID from jwt token")
}
secret, err := j.keyStore.Get(siteID)
secret, err := j.keyStore.Key(siteID)
if err != nil {
return nil, errors.Wrap(err, "can't get secret")
}
backend/app/rest/auth/jwt_test.go
+8 -8
View File
@@ -10,7 +10,7 @@ import (
"github.com/dgrijalva/jwt-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store"
)
@@ -30,7 +30,7 @@ var testJwtBadSign = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4M
var days31 = time.Hour * 24 * 31
func TestJWT_Token(t *testing.T) {
j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -53,7 +53,7 @@ func TestJWT_Token(t *testing.T) {
}
func TestJWT_Parse(t *testing.T) {
j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims, err := j.Parse(testJwtValid)
assert.NoError(t, err)
assert.False(t, j.IsExpired(claims))
@@ -71,7 +71,7 @@ func TestJWT_Parse(t *testing.T) {
}
func TestJWT_Set(t *testing.T) {
j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -116,7 +116,7 @@ func TestJWT_Set(t *testing.T) {
}
func TestJWT_GetFromHeader(t *testing.T) {
j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
req := httptest.NewRequest("GET", "/", nil)
req.Header.Add(jwtHeaderKey, testJwtValid)
@@ -141,7 +141,7 @@ func TestJWT_GetFromHeader(t *testing.T) {
}
func TestJWT_SetAndGetWithCookies(t *testing.T) {
j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -183,7 +183,7 @@ func TestJWT_SetAndGetWithCookies(t *testing.T) {
}
func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) {
j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -220,7 +220,7 @@ func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) {
}
func TestJWT_SetAndGetWithCookiesExpired(t *testing.T) {
j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
backend/app/rest/auth/provider_test.go
+4 -4
View File
@@ -13,10 +13,10 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/keys"
"golang.org/x/oauth2"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
)
func TestLogin(t *testing.T) {
@@ -54,7 +54,7 @@ func TestLogin(t *testing.T) {
Admin: false, Blocked: true, IP: ""}, u)
token := resp.Cookies()[0].Value
jwtSvc := NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31)
jwtSvc := NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31)
claims, err := jwtSvc.Parse(token)
require.NoError(t, err)
@@ -103,7 +103,7 @@ func TestLoginSessionOnly(t *testing.T) {
req.AddCookie(resp.Cookies()[1])
req.Header.Add("X-XSRF-TOKEN", resp.Cookies()[1].Value)
jwtService := NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour)
jwtService := NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour)
res, err := jwtService.Get(req)
require.Nil(t, err)
assert.Equal(t, true, res.SessionOnly)
@@ -169,7 +169,7 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se
}
params := Params{RemarkURL: "url", Cid: "cid", Csecret: "csecret",
JwtService: NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31),
JwtService: NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31),
// AvatarProxy: &proxy.Avatar{Store: &mockAvatarStore, RoutePath: "/v1/avatar"},
PermissionChecker: &mockUserPermissions{admin: "mock_myuser2", verified: "mock_myuser2", blocked: "mock_myuser1"},
}
backend/app/store/admin/admin.go
+19 -49
View File
@@ -1,15 +1,14 @@
// Package admin defines and implements store for admin-level data like secret key, list of admins and so on
package admin
import (
"errors"
"log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
)
// Store defines interface returning admins info for given site
type Store interface {
Key(siteID string) (key string, err error)
Admins(siteID string) (ids []string)
Email(siteID string) (email string)
}
@@ -18,12 +17,26 @@ type Store interface {
type StaticStore struct {
admins []string
email string
key string
}
// Key returns static key for all sites, allows empty site
func (s *StaticStore) Key(siteID string) (key string, err error) {
if s.key == "" {
return "", errors.New("empty key for static key store")
}
return s.key, nil
}
// NewStaticStore makes StaticStore instance with given key
func NewStaticStore(admins []string, email string) *StaticStore {
func NewStaticStore(key string, admins []string, email string) *StaticStore {
log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
return &StaticStore{admins: admins, email: email}
return &StaticStore{key: key, admins: admins, email: email}
}
// NewStaticKeyStore is a shortcut for making StaticStore for key consumers only
func NewStaticKeyStore(key string) *StaticStore {
return &StaticStore{key: key, admins: []string{}, email: ""}
}
// Admins returns static list of admin's ids, the same for all sites
@@ -35,46 +48,3 @@ func (s *StaticStore) Admins(string) (ids []string) {
func (s *StaticStore) Email(string) (email string) {
return s.email
}
// MongoStore implements admin.Store with mongo backend
type MongoStore struct {
connection *mongo.Connection
}
// NewMongoStore makes admin Store for mongo's connection
func NewMongoStore(conn *mongo.Connection) *MongoStore {
log.Printf("[DEBUG] make mongo admin store with %+v", conn)
return &MongoStore{connection: conn}
}
// Admins executes find by siteID and returns admins ids
func (m *MongoStore) Admins(siteID string) (ids []string) {
resp := struct {
SiteID string `bson:"site"`
IDs []string `bson:"admin_ids"`
Email string `bson:"admin_email"`
}{}
err := m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
if err != nil {
return []string{}
}
return resp.IDs
}
// Email executes find by siteID and returns admin's email
func (m *MongoStore) Email(siteID string) (email string) {
resp := struct {
SiteID string `bson:"site"`
IDs []string `bson:"admin_ids"`
Email string `bson:"admin_email"`
}{}
err := m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
if err != nil {
return ""
}
return resp.Email
}
backend/app/store/admin/admin_test.go
+20 -8
View File
@@ -10,13 +10,21 @@ import (
)
func TestStaticStore_Get(t *testing.T) {
var ks Store = NewStaticStore([]string{"123", "xyz"}, "aa@example.com")
var ks Store = NewStaticStore("key123", []string{"123", "xyz"}, "aa@example.com")
k, err := ks.Key("any")
assert.NoError(t, err, "valid store")
assert.Equal(t, "key123", k, "valid site")
a := ks.Admins("any")
assert.Equal(t, []string{"123", "xyz"}, a)
email := ks.Email("blah")
assert.Equal(t, "aa@example.com", email)
ks = NewStaticStore("", []string{"123", "xyz"}, "aa@example.com")
_, err = ks.Key("any")
assert.NotNil(t, err, "invalid (empty key) store")
}
func TestMongoStore_Get(t *testing.T) {
@@ -24,13 +32,9 @@ func TestMongoStore_Get(t *testing.T) {
require.NoError(t, err)
var ms Store = NewMongoStore(conn)
recs := []struct {
SiteID string `bson:"site"`
IDs []string `bson:"admin_ids"`
Email string `bson:"admin_email"`
}{
{"site1", []string{"i11", "i12"}, "e1"},
{"site2", []string{"i21", "i22"}, "e2"},
recs := []mongoRec{
{"site1", "secret1", []string{"i11", "i12"}, "e1"},
{"site2", "secret2", []string{"i21", "i22"}, "e2"},
}
err = conn.WithCollection(func(coll *mgo.Collection) error {
if e1 := coll.Insert(recs[0]); e1 != nil {
@@ -47,14 +51,22 @@ func TestMongoStore_Get(t *testing.T) {
assert.Equal(t, []string{"i11", "i12"}, admins)
email := ms.Email("site1")
assert.Equal(t, "e1", email)
key, err := ms.Key("site1")
assert.NoError(t, err)
assert.Equal(t, "secret1", key)
admins = ms.Admins("site2")
assert.Equal(t, []string{"i21", "i22"}, admins)
email = ms.Email("site2")
assert.Equal(t, "e2", email)
key, err = ms.Key("site2")
assert.NoError(t, err)
assert.Equal(t, "secret2", key)
admins = ms.Admins("no-site-in-db")
assert.Equal(t, []string{}, admins)
email = ms.Email("no-site-in-db")
assert.Equal(t, "", email)
_, err = ms.Key("no-site-in-db")
assert.Error(t, err, "can't get secret for site no-site-in-db")
}
backend/app/store/admin/mongo.go
+61
View File
@@ -0,0 +1,61 @@
package admin
import (
"log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
)
// MongoStore implements admin.Store with mongo backend
type MongoStore struct {
connection *mongo.Connection
}
type mongoRec struct {
SiteID string `bson:"site"`
SecretKey string `bson:"secret"`
IDs []string `bson:"admin_ids"`
Email string `bson:"admin_email"`
}
// NewMongoStore makes admin Store for mongo's connection
func NewMongoStore(conn *mongo.Connection) *MongoStore {
log.Printf("[DEBUG] make mongo admin store with %+v", conn)
return &MongoStore{connection: conn}
}
// Key executes find by siteID and returns substructure with secret key
func (m *MongoStore) Key(siteID string) (key string, err error) {
resp := mongoRec{}
err = m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
return resp.SecretKey, errors.Wrapf(err, "can't get secret for site %s", siteID)
}
// Admins executes find by siteID and returns admins ids
func (m *MongoStore) Admins(siteID string) (ids []string) {
resp := mongoRec{}
err := m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
if err != nil {
return []string{}
}
return resp.IDs
}
// Email executes find by siteID and returns admin's email
func (m *MongoStore) Email(siteID string) (email string) {
resp := mongoRec{}
err := m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
if err != nil {
return ""
}
return resp.Email
}
backend/app/store/avatar/gridfs.go
+24 -1
View File
@@ -5,6 +5,7 @@ import (
"io"
"io/ioutil"
"log"
"time"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
@@ -26,7 +27,7 @@ type GridFS struct {
// Put avatar to gridfs object, try to resize
func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := store.EncodeID(userID)
id := encodeID(userID)
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Create(id + imgSfx)
if e != nil {
@@ -95,3 +96,25 @@ func (gf *GridFS) Remove(avatar string) error {
return dbase.GridFS("fs").Remove(avatar)
})
}
// List all avatars (ids) on gfs
// note: id includes .image suffix
func (gf *GridFS) List() (ids []string, err error) {
type gfsFile struct {
UploadDate time.Time `bson:"uploadDate"`
Length int64 `bson:",minsize"`
MD5 string
Filename string `bson:",omitempty"`
}
files := []gfsFile{}
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
return dbase.GridFS("fs").Find(nil).All(&files)
})
for _, f := range files {
ids = append(ids, f.Filename)
}
return ids, errors.Wrap(err, "can't list avatars")
}
backend/app/store/avatar/gridfs_test.go
+33
View File
@@ -2,6 +2,7 @@ package avatar
import (
"io/ioutil"
"sort"
"strings"
"testing"
@@ -32,6 +33,11 @@ func TestGridFS_PutAndGet(t *testing.T) {
assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar))
assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id")
l, err := p.List()
require.Nil(t, err)
assert.Equal(t, 1, len(l))
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0])
}
func TestGridFS_Remove(t *testing.T) {
@@ -49,6 +55,33 @@ func TestGridFS_Remove(t *testing.T) {
assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed")
}
func TestGridFS_List(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
// write some avatars
_, err := p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func prepGFStore(t *testing.T) (Store, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
backend/app/store/avatar/localfs.go
+19 -1
View File
@@ -7,6 +7,7 @@ import (
"log"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
@@ -30,8 +31,9 @@ func NewLocalFS(storePath string, resizeLimit int) *LocalFS {
}
// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
// userID can be avatarID as well, in this case encoding just strip .image prefix
func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := store.EncodeID(userID)
id := encodeID(userID)
location := fs.location(id) // location adds partition to path
if _, err = os.Stat(location); os.IsNotExist(err) {
@@ -95,6 +97,22 @@ func (fs *LocalFS) Remove(avatar string) error {
return os.Remove(avFile)
}
// List all avatars (ids) on local file system
// note: id includes .image suffix
func (fs *LocalFS) List() (ids []string, err error) {
err = filepath.Walk(fs.storePath,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasSuffix(info.Name(), imgSfx) {
ids = append(ids, info.Name())
}
return nil
})
return ids, errors.Wrap(err, "can't list avatars")
}
// get location (directory) for user id by adding partition to final path in order to keep files
// in different subdirectories and avoid too many files in a single place.
// the end result is a full path like this - /tmp/avatars.test/92
backend/app/store/avatar/localfs_test.go
+38
View File
@@ -3,6 +3,7 @@ package avatar
import (
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"time"
@@ -35,6 +36,14 @@ func TestAvatarStoreFS_Put(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with encoded id
avatar, err = p.Put("f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", strings.NewReader("some picture bin data 123"))
require.Nil(t, err)
assert.Equal(t, "f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", avatar)
fi, err = os.Stat("/tmp/avatars.test/56/f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image")
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with resize
file, e := os.Open("testdata/circles.png")
require.Nil(t, e)
@@ -84,6 +93,7 @@ func TestAvatarStoreFS_Location(t *testing.T) {
{"abc", "/tmp/avatars.test/35"},
{"xyz", "/tmp/avatars.test/69"},
{"blah blah", "/tmp/avatars.test/29"},
{"f1881c06eec96db9901c7bbfe41c42a3f08e9cb8", "/tmp/avatars.test/56"},
}
for i, tt := range tbl {
@@ -126,6 +136,34 @@ func TestAvatarStoreFS_Remove(t *testing.T) {
t.Log(err)
}
func TestAvatarStoreFS_List(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// write some avatars
_, err = p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func BenchmarkAvatarStoreFS_ID(b *testing.B) {
p := NewLocalFS("/tmp/avatars.test", 300)
os.MkdirAll("/tmp/avatars.test/30", 0700)
backend/app/store/avatar/store.go
+39 -3
View File
@@ -6,29 +6,57 @@ package avatar
import (
"bytes"
"image"
"image/png"
"io"
"log"
"strings"
// Initializing packages for supporting GIF and JPEG formats.
_ "image/gif"
_ "image/jpeg"
"image/png"
"io"
"log"
"regexp"
"github.com/umputun/remark/backend/app/store"
"golang.org/x/image/draw"
)
// imgSfx for avatars
const imgSfx = ".image"
var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`)
// Store defines interface to store and and load avatars
type Store interface {
Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name
Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader
ID(avatarID string) (id string) // unique id of stored avatar's data
Remove(avatarID string) error // remove avatar data
List() (ids []string, err error) // list all avatar ids
}
// Migrate avatars between stores
func Migrate(dst Store, src Store) (int, error) {
ids, err := src.List()
if err != nil {
return 0, err
}
for _, id := range ids {
srcReader, _, err := src.Get(id)
if err != nil {
log.Printf("[WARN] can't get reader for avatar %s", id)
continue
}
if _, err = dst.Put(id, srcReader); err != nil {
log.Printf("[WARN] can't put avatar %s", id)
}
if err = srcReader.Close(); err != nil {
log.Printf("[WARN] failed to close avatar %s", id)
}
}
return len(ids), nil
}
// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
// (width or height) preserving aspect ratio.
// Returns original reader if resizing is not needed or failed.
@@ -71,3 +99,11 @@ func resize(reader io.Reader, limit int) io.Reader {
}
return &out
}
// encodeID converts string to encoded id unless already encoded and valid avatar id (with .image) passed
func encodeID(val string) string {
if reValidAvatarID.MatchString(val) {
return strings.TrimSuffix(val, imgSfx) // already encoded, strip .image
}
return store.EncodeID(val)
}
backend/app/store/avatar/store_test.go
+44
View File
@@ -5,6 +5,8 @@ import (
"image"
"io"
"io/ioutil"
"os"
"sort"
"strings"
"testing"
@@ -63,3 +65,45 @@ func TestAvatarStore_resize(t *testing.T) {
assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
}
}
func TestAvatarStore_Migrate(t *testing.T) {
// prep localfs
plocal := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// prep gridfs
pgfs, skip := prepGFStore(t)
if skip {
return
}
// write to localfs
_, err = plocal.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = plocal.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = plocal.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
// migrate and check reported count
count, err := Migrate(pgfs, plocal)
require.NoError(t, err)
assert.Equal(t, 3, count, "all 3 recs migrated")
// list avatars
l, err := pgfs.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed in destination store")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
// try to read one of migrated avatars
r, size, err := pgfs.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
backend/app/store/keys/keys.go
-57
View File
@@ -1,57 +0,0 @@
package keys
import (
"log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
)
// Store defines interface returning key for given site
// this key used for JWT and HMAC hashes
type Store interface {
Get(siteID string) (key string, err error)
}
// StaticStore implements keys.Store with a single, predefined key
type StaticStore struct {
key string
}
// NewStaticStore makes StaticStore instance with given key
func NewStaticStore(key string) *StaticStore {
return &StaticStore{key: key}
}
// Get returns static key for all sites, allows empty site
func (s *StaticStore) Get(siteID string) (key string, err error) {
if s.key == "" {
return "", errors.New("empty key for static key store")
}
return s.key, nil
}
// MongoStore implements keys.Store with mongo backend
type MongoStore struct {
connection *mongo.Connection
}
// NewMongoStore makes keys Store for mongo's connection
func NewMongoStore(conn *mongo.Connection) *MongoStore {
log.Printf("[DEBUG] make mongo keys store with %+v", conn)
return &MongoStore{connection: conn}
}
// Get executes find by siteID and returns substructure with secret key
func (m *MongoStore) Get(siteID string) (key string, err error) {
resp := struct {
SiteID string `bson:"site"`
SecretKey string `bson:"secret"`
}{}
err = m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
return resp.SecretKey, errors.Wrapf(err, "can't get secret for site %s", siteID)
}
backend/app/store/keys/keys_test.go
-58
View File
@@ -1,58 +0,0 @@
package keys
import (
"testing"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestStaticStore_Get(t *testing.T) {
var ks Store = NewStaticStore("key123")
k, err := ks.Get("any")
assert.NoError(t, err, "valid store")
assert.Equal(t, "key123", k, "valid site")
ks = NewStaticStore("")
_, err = ks.Get("any")
assert.NotNil(t, err, "invalid (empty key) store")
}
func TestMongoStore_Get(t *testing.T) {
conn, err := mongo.MakeTestConnection(t)
require.NoError(t, err)
var ms Store = NewMongoStore(conn)
recs := []struct {
SiteID string `bson:"site"`
SecretKey string `bson:"secret"`
}{
{"site1", "secret1"},
{"site2", "secret2"},
}
err = conn.WithCollection(func(coll *mgo.Collection) error {
if e1 := coll.Insert(recs[0]); e1 != nil {
return e1
}
if e2 := coll.Insert(recs[1]); e2 != nil {
return e2
}
return nil
})
require.NoError(t, err)
r, err := ms.Get("site1")
assert.NoError(t, err)
assert.Equal(t, "secret1", r)
r, err = ms.Get("site2")
assert.NoError(t, err)
assert.Equal(t, "secret2", r)
_, err = ms.Get("no-site-in-db")
assert.Error(t, err, "can't get secret for site no-site-in-db")
}
backend/app/store/service/service.go
+2 -4
View File
@@ -6,18 +6,16 @@ import (
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
)
// DataStore wraps store.Interface with additional methods
type DataStore struct {
engine.Interface
EditDuration time.Duration
KeyStore keys.Store
AdminStore admin.Store
MaxCommentSize int
@@ -56,7 +54,7 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err
}
comment.Sanitize() // clear potentially dangerous js from all parts of comment
secret, err := s.KeyStore.Get(comment.Locator.SiteID)
secret, err := s.AdminStore.Key(comment.Locator.SiteID)
if err != nil {
return store.Comment{}, errors.Wrapf(err, "can't get secret for site %s", comment.Locator.SiteID)
}
backend/app/store/service/service_test.go
+13 -13
View File
@@ -13,7 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
@@ -23,8 +23,8 @@ var testDb = "/tmp/test-remark.db"
func TestService_CreateFromEmpty(t *testing.T) {
defer os.Remove(testDb)
ks := keys.NewStaticStore("secret 123")
b := DataStore{Interface: prepStoreEngine(t), KeyStore: ks}
ks := admin.NewStaticKeyStore("secret 123")
b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
comment := store.Comment{
Text: "text",
User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
@@ -47,8 +47,8 @@ func TestService_CreateFromEmpty(t *testing.T) {
func TestService_CreateFromPartial(t *testing.T) {
defer os.Remove(testDb)
ks := keys.NewStaticStore("secret 123")
b := DataStore{Interface: prepStoreEngine(t), KeyStore: ks}
ks := admin.NewStaticKeyStore("secret 123")
b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
comment := store.Comment{
Text: "text",
Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
@@ -73,7 +73,7 @@ func TestService_CreateFromPartial(t *testing.T) {
func TestService_Vote(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
comment := store.Comment{
Text: "text",
@@ -118,7 +118,7 @@ func TestService_Vote(t *testing.T) {
func TestService_VoteAggressive(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
comment := store.Comment{
Text: "text",
@@ -178,7 +178,7 @@ func TestService_VoteAggressive(t *testing.T) {
func TestService_VoteConcurrent(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
comment := store.Comment{
Text: "text",
@@ -209,7 +209,7 @@ func TestService_VoteConcurrent(t *testing.T) {
func TestService_Pin(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
@@ -233,7 +233,7 @@ func TestService_Pin(t *testing.T) {
func TestService_EditComment(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
@@ -260,7 +260,7 @@ func TestService_EditComment(t *testing.T) {
func TestService_DeleteComment(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
@@ -279,7 +279,7 @@ func TestService_DeleteComment(t *testing.T) {
func TestService_EditCommentDurationFailed(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
@@ -296,7 +296,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
func TestService_ValidateComment(t *testing.T) {
b := DataStore{MaxCommentSize: 2000, KeyStore: keys.NewStaticStore("secret 123")}
b := DataStore{MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")}
longText := fmt.Sprintf("%4000s", "X")
tbl := []struct {
compose-dev-backend.yml
+1 -1
View File
@@ -1,5 +1,5 @@
# compose file for local development
# starts backend on 8080 with basic auth "dev:password" and Dev oauth2 provider on port 8084, UI on https://127.0.0.1:8080/web
# starts backend on 8080 with basic auth "dev:password" and Dev oauth2 provider on port 8084, UI on http://127.0.0.1:8080/web
#
# mongo-related tests needs mongodb container running - docker run -d -name=mongo mongo:3.6 --smallfiles
# start build with backend tests: