package cmd import ( "context" "fmt" "log" "net/url" "os" "os/signal" "strings" "syscall" "time" "github.com/coreos/bbolt" "github.com/go-pkgz/mongo" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/api" "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) // ServerOpts with command line flags and env type ServerOpts struct { RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"` DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"` BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"` MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"` MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"` LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"` CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"` ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"` EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"` Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` // Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` Auth struct { TTL struct { JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"` Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"` } `group:"ttl" namespace:"ttl" env-namespace:"TTL"` Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"` Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"` Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"` Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"` Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"` } `group:"auth" namespace:"auth" env-namespace:"AUTH"` } // AuthGroup defines options group for auth params type AuthGroup struct { CID string `long:"cid" env:"CID" description:"OAuth client ID"` CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"` } // StoreGroup defines options group for store params type StoreGroup struct { Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"` Bolt struct { Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` } // AvatarGroup defines options group for avatar params type AvatarGroup struct { Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"` FS struct { Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` } `group:"fs" namespace:"fs" env-namespace:"FS"` RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` } // CacheGroup defines options group for cache params type CacheGroup struct { Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"` Max struct { Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"` } `group:"max" namespace:"max" env-namespace:"MAX"` } // MongoGroup holds all mongo params, used by store, avatar and cache type MongoGroup struct { URL string `long:"url" env:"URL" description:"mongo url"` DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` } // KeyGroup defines options group for key params type KeyGroup struct { Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` } // Revision sets from main var Revision = "unknown" // serverApp holds all active objects type serverApp struct { *ServerOpts restSrv *api.Rest migratorSrv *api.Migrator exporter migrator.Exporter devAuth *auth.DevAuthServer dataService *service.DataStore terminated chan struct{} } // Execute is the entry point for "server" command, called by flag parser func (s *ServerOpts) Execute(args []string) error { log.Print("[INFO] start remark42 server") resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC") ctx, cancel := context.WithCancel(context.Background()) go func() { // catch signal and invoke graceful termination stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) <-stop log.Print("[WARN] interrupt signal") cancel() }() app, err := newServerApp(s) if err != nil { log.Fatalf("[ERROR] failed to setup application, %+v", err) } if err = app.run(ctx); err != nil { log.Printf("[INFO] remark terminated with error %+v", err) return err } log.Printf("[INFO] remark terminated") return nil } // newServerApp prepares application and return it with all active parts // doesn't start anything func newServerApp(opts *ServerOpts) (*serverApp, error) { if err := makeDirs(opts.BackupLocation); err != nil { return nil, err } if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") { return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL) } storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites) if err != nil { return nil, err } keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret) if err != nil { return nil, err } dataService := &service.DataStore{ Interface: storeEngine, EditDuration: opts.EditDuration, KeyStore: keyStore, MaxCommentSize: opts.MaxCommentSize, Admins: opts.Admins, } loadingCache, err := makeCache(opts.Cache, opts.Mongo) if err != nil { return nil, err } // token TTL is 5 minutes, inactivity interval 7+ days by default jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) if err != nil { return nil, errors.Wrap(err, "failed to make avatar store") } avatarProxy := &proxy.Avatar{ Store: avatarStore, RoutePath: "/api/v1/avatar", RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), } exporter := &migrator.Remark{DataStore: dataService} migr := &api.Migrator{ Version: Revision, Cache: loadingCache, NativeImporter: &migrator.Remark{DataStore: dataService}, DisqusImporter: &migrator.Disqus{DataStore: dataService}, WordPressImporter: &migrator.WordPress{DataStore: dataService}, NativeExported: &migrator.Remark{DataStore: dataService}, KeyStore: keyStore, } authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} commentFormatter := store.NewCommentFormatter(imgProxy) srv := &api.Rest{ Version: Revision, DataService: dataService, Exporter: exporter, WebRoot: opts.WebRoot, RemarkURL: opts.RemarkURL, ImageProxy: imgProxy, CommentFormatter: commentFormatter, AvatarProxy: avatarProxy, ReadOnlyAge: opts.ReadOnlyAge, SharedSecret: opts.SharedSecret, Authenticator: auth.Authenticator{ JWTService: jwtService, AdminEmail: opts.AdminEmail, Providers: authProviders, DevPasswd: opts.DevPasswd, PermissionChecker: dataService, }, Cache: loadingCache, } // no admin email, use admin@domain if srv.Authenticator.AdminEmail == "" { if u, err := url.Parse(opts.RemarkURL); err == nil { srv.Authenticator.AdminEmail = "admin@" + u.Host } } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore var devAuth *auth.DevAuthServer if opts.Auth.Dev { devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]} } tch := make(chan struct{}) return &serverApp{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService, ServerOpts: opts, terminated: tch}, nil } // Run all application objects func (a *serverApp) run(ctx context.Context) error { if a.DevPasswd != "" { log.Printf("[WARN] running in dev mode") } go func() { // shutdown on context cancellation <-ctx.Done() a.restSrv.Shutdown() a.migratorSrv.Shutdown() if a.devAuth != nil { a.devAuth.Shutdown() } if e := a.dataService.Close(); e != nil { log.Printf("[WARN] failed to close store, %s", e) } }() a.activateBackup(ctx) // runs in goroutine for each site go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only if a.Auth.Dev { go a.devAuth.Run() // dev oauth2 server on :8084 } a.restSrv.Run(a.Port) close(a.terminated) return nil } // Wait for application completion (termination) func (a *serverApp) Wait() { <-a.terminated } // activateBackup runs background backups for each site func (a *serverApp) activateBackup(ctx context.Context) { for _, siteID := range a.Sites { backup := migrator.AutoBackup{ Exporter: a.exporter, BackupLocation: a.BackupLocation, SiteID: siteID, KeepMax: a.MaxBackupFiles, Duration: 24 * time.Hour, } go backup.Do(ctx) } } // makeDataStore creates store for all sites func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) { switch group.Type { case "bolt": if err = makeDirs(group.Bolt.Path); err != nil { return nil, errors.Wrap(err, "failed to create bolt store") } sites := []engine.BoltSite{} for _, site := range siteNames { sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)}) } result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...) case "mongo": mgServer, e := makeMongo(mg) if e != nil { return result, errors.Wrap(e, "failed to create mongo server") } conn := mongo.NewConnection(mgServer, mg.DB, "") result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) default: return nil, errors.Errorf("unsupported store type %s", group.Type) } return result, errors.Wrap(err, "can't initialize data store") } func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { switch group.Type { case "fs": if err := makeDirs(group.FS.Path); err != nil { return nil, err } return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil case "mongo": mgServer, err := makeMongo(mg) if err != nil { return nil, errors.Wrap(err, "failed to create mongo server") } conn := mongo.NewConnection(mgServer, mg.DB, "") return avatar.NewGridFS(conn, group.RszLmt), nil } return nil, errors.Errorf("unsupported avatar store type %s", group.Type) } func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) { switch group.Type { case "shared": return keys.NewStaticStore(sharedSecret), nil default: return nil, errors.Errorf("unsupported key store type %s", group.Type) } } func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { switch group.Type { case "mem": return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), cache.MaxKeys(group.Max.Items)) case "mongo": mgServer, err := makeMongo(mg) if err != nil { return nil, errors.Wrap(err, "failed to create mongo server") } conn := mongo.NewConnection(mgServer, mg.DB, "cache") return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), cache.MaxKeys(group.Max.Items)) } return nil, errors.Errorf("unsupported cache type %s", group.Type) } // mkdir -p for all dirs func makeDirs(dirs ...string) error { // exists returns whether the given file or directory exists or not exists := func(path string) (bool, error) { _, err := os.Stat(path) if err == nil { return true, nil } if os.IsNotExist(err) { return false, nil } return true, err } for _, dir := range dirs { ex, err := exists(dir) if err != nil { return errors.Wrapf(err, "can't check directory status for %s", dir) } if !ex { if e := os.MkdirAll(dir, 0700); e != nil { return errors.Wrapf(err, "can't make directory %s", dir) } } } return nil } func makeMongo(mg MongoGroup) (result *mongo.Server, err error) { if mg.URL == "" { return nil, errors.New("no mongo URL provided") } return mongo.NewServerWithURL(mg.URL, 10*time.Second) } func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts *ServerOpts) []auth.Provider { makeParams := func(cid, secret string) auth.Params { return auth.Params{ JwtService: jwtService, AvatarProxy: avatarProxy, RemarkURL: opts.RemarkURL, Cid: cid, Csecret: secret, PermissionChecker: ds, } } providers := []auth.Provider{} if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" { providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC))) } if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" { providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC))) } if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" { providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC))) } if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" { providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC))) } if opts.Auth.Dev { providers = append(providers, auth.NewDev(makeParams("", ""))) } if len(providers) == 0 { log.Printf("[WARN] no auth providers defined") } return providers }