mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #84 from umputun/feature/store

Browse Source 
Feature/store
This commit is contained in:
Umputun
2018-06-17 13:05:25 -05:00
committed by GitHub
parent 835606c5cf 7bebc5ddfc
commit 1c0e1a2b07
3 changed files with 96 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7 -6
View File
@@ -35,17 +35,18 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
| ------------------ | ------------------ | --------------------- | ---------------------------------------------- |
| url | REMARK_URL | | url to remark42 server, _required_ |
| secret | SECRET | | secret key, _required_ |
| bolt | BOLTDB_PATH | `./var` | path to data directory |
| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory |
| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout |
| site | SITE | `remark` | site name(s), _multi_ |
| admin | ADMIN | | admin names (list of user ids), _multi_ |
| admin-email | ADMIN_EMAIL | `admin@${REMARK_URL}` | admin email |
| backup | BACKUP_PATH | `./var/backup` | backups location |
| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep |
| max-cache-items | MAX_CACHE_ITEMS | `1000` | max number of cached items, `0` - unlimited |
| max-cache-value | MAX_CACHE_VALUE | `65536` | max size of cached value, `0` - unlimited |
| max-cache-size | MAX_CACHE_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
| avatars | AVATAR_STORE | `./var/avatars` | avatars location |
| avatars-rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save |
| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited |
| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited |
| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
| avatar.path | AVATAR_FS_PATH | `./var/avatars` | avatars location |
| avatar.rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save |
| max-comment | MAX_COMMENT_SIZE | 2048 | comment's size limit |
| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID |
| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret |
app/main.go
+67 -20
View File
@@ -32,20 +32,18 @@ type Opts struct {
SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
BoltPath string `long:"bolt" env:"BOLTDB_PATH" default:"./var" description:"parent dir for bolt files"`
Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"`
Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"`
Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"`
DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"avatars location"`
AvatarRszLmt int `long:"avatars-rsz-lmt" env:"AVATAR_RSZ_LMT" default:"0" description:"max image size for resizing avatars on save"`
ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
MaxCachedItems int `long:"max-cache-items" env:"MAX_CACHE_ITEMS" default:"1000" description:"max cached items"`
MaxCachedValue int `long:"max-cache-value" env:"MAX_CACHE_VALUE" default:"65536" description:"max size of cached value"`
MaxCacheSize int `long:"max-cache-size" env:"MAX_CACHE_SIZE" default:"50000000" description:"max size of total cache"`
LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
@@ -67,6 +65,34 @@ type AuthGroup struct {
CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
}
// StoreGroup defines options group for store params
type StoreGroup struct {
Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
Bolt struct {
Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
}
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
RszLmt int `long:"rsz-lmt" env:"RSZ_LMT" default:"0" description:"max image size for resizing avatars on save"`
}
// CacheGroup defines options group for cache params
type CacheGroup struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"redis" default:"mem"`
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
} `group:"max" namespace:"max" env-namespace:"MAX"`
}
var revision = "unknown"
// Application holds all active objects
@@ -112,7 +138,7 @@ func main() {
// doesn't start anything
func New(opts Opts) (*Application, error) {
if err := makeDirs(opts.BoltPath, opts.BackupLocation, opts.AvatarStore); err != nil {
if err := makeDirs(opts.BackupLocation); err != nil {
return nil, err
}
@@ -120,7 +146,7 @@ func New(opts Opts) (*Application, error) {
return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
}
boltStore, err := makeBoltStore(opts.Sites, opts.BoltPath)
boltStore, err := makeDataStore(opts.Store, opts.Sites)
if err != nil {
return nil, err
}
@@ -131,16 +157,20 @@ func New(opts Opts) (*Application, error) {
MaxCommentSize: opts.MaxCommentSize,
}
loadingCache, err := cache.NewMemoryCache(cache.MaxValSize(opts.MaxCachedValue), cache.MaxKeys(opts.MaxCachedItems),
cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port)))
loadingCache, err := cache.NewMemoryCache(cache.MaxCacheSize(opts.Cache.Max.Size), cache.MaxValSize(opts.Cache.Max.Value),
cache.MaxKeys(opts.Cache.Max.Items), cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port)))
if err != nil {
return nil, err
}
jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour)
avatarStore, err := makeAvatarStore(opts.Avatar)
if err != nil {
return nil, errors.Wrap(err, "failed to make avatar store")
}
avatarProxy := &proxy.Avatar{
Store: proxy.NewFSAvatarStore(opts.AvatarStore, opts.AvatarRszLmt),
Store: avatarStore,
RoutePath: "/api/v1/avatar",
RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
}
@@ -225,17 +255,34 @@ func (a *Application) activateBackup(ctx context.Context) {
}
}
// makeBoltStore creates store for all sites
func makeBoltStore(siteNames []string, path string) (engine.Interface, error) {
sites := []engine.BoltSite{}
for _, site := range siteNames {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", path, site)})
// makeDataStore creates store for all sites
func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interface, err error) {
switch group.Type {
case "bolt":
if err = makeDirs(group.Bolt.Path); err != nil {
return nil, err
}
sites := []engine.BoltSite{}
for _, site := range siteNames {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
default:
return nil, errors.Errorf("unsupported store type %s", group.Type)
}
result, err := engine.NewBoltDB(bolt.Options{Timeout: 30 * time.Second}, sites...)
if err != nil {
return nil, errors.Wrap(err, "can't initialize data store")
return result, errors.Wrap(err, "can't initialize data store")
}
func makeAvatarStore(group AvatarGroup) (result proxy.AvatarStore, err error) {
switch group.Type {
case "fs":
if err = makeDirs(group.FS.Path); err != nil {
return nil, err
}
return proxy.NewFSAvatarStore(group.FS.Path, group.RszLmt), nil
}
return result, nil
return nil, errors.Errorf("unsupported avatart store type %s", group.Type)
}
// mkdir -p for all dirs
app/main_test.go
+22 -9
View File
@@ -49,25 +49,36 @@ func TestApplicationFailed(t *testing.T) {
p := flags.NewParser(&opts, flags.Default)
// RO bolt location
p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--bolt=/dev/null"})
_, err := New(opts)
_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/dev/null"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
"open /dev/null/remark.db: not a directory")
t.Log(err)
// RO backup location
opts = Opts{}
p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--bolt=/tmp", "--backup=/dev/null/not-writable"})
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
"--backup=/dev/null/not-writable"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
t.Log(err)
// invalid url
opts = Opts{}
p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--bolt=/tmp"})
_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "----store.bolt.path=/tmp"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
t.Log(err)
opts = Opts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.type=mongo"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "unsupported store type mongo")
t.Log(err)
}
func TestApplicationShutdown(t *testing.T) {
@@ -79,7 +90,7 @@ func TestApplicationShutdown(t *testing.T) {
}
func TestApplicationMainSignal(t *testing.T) {
os.Args = []string{"test", "--secret=123456", "--bolt=/tmp/xyz", "--backup=/tmp", "--avatars=/tmp",
os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
"--port=18100", "--url=https://demo.remark42.com"}
go func() {
@@ -95,16 +106,18 @@ func prepApp(t *testing.T, port int, duration time.Duration) (*Application, cont
// prepare options
opts := Opts{}
p := flags.NewParser(&opts, flags.Default)
p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
opts.AvatarStore, opts.BackupLocation = "/tmp", "/tmp"
opts.BoltPath = fmt.Sprintf("/tmp/%d", port)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
require.Nil(t, err)
opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp"
opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", port)
opts.Store.Bolt.Timeout = 10 * time.Second
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid"
opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid"
opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid"
opts.Port = port
os.Remove(opts.BoltPath + "/remark.db")
os.Remove(opts.Store.Bolt.Path + "/remark.db")
// create app
app, err := New(opts)