diff --git a/app/main.go b/app/main.go index a8de035d..6cb4ce37 100644 --- a/app/main.go +++ b/app/main.go @@ -26,6 +26,7 @@ var opts struct { Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"/tmp" description:"backups location"` + MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` ServerCommand struct { SessionStore string `long:"session" env:"SESSION_STORE" default:"/tmp" description:"path to session store directory"` @@ -112,7 +113,10 @@ func main() { log.Printf("[WARN] running in dev mode, no auth!") } - go migrator.AutoBackup(&exporter, opts.BackupLocation) + for _, siteID := range opts.Sites { + go migrator.AutoBackup(&exporter, opts.BackupLocation, siteID, opts.MaxBackupFiles) + } + srv.Run() } diff --git a/app/migrator/migrator.go b/app/migrator/migrator.go index 412d50ce..baeb1243 100644 --- a/app/migrator/migrator.go +++ b/app/migrator/migrator.go @@ -4,8 +4,11 @@ import ( "compress/gzip" "fmt" "io" + "io/ioutil" "log" "os" + "sort" + "strings" "time" "github.com/pkg/errors" @@ -59,22 +62,50 @@ func ImportComments(p ImportParams) error { } // AutoBackup runs daily export to local files -func AutoBackup(exporter Exporter, backupLocation string) { +func AutoBackup(exporter Exporter, backupLocation string, siteID string, keepMax int) { log.Print("[INFO] activate auto-backup") tick := time.NewTicker(24 * time.Hour) for range tick.C { log.Print("[DEBUG] make backup") - fh, err := os.Create(fmt.Sprintf("%s/backup-%s.gz", backupLocation, time.Now().Format("20060102"))) + fh, err := os.Create(fmt.Sprintf("%s/backup-%s-%s.gz", backupLocation, siteID, time.Now().Format("20060102"))) if err != nil { log.Printf("[WARN] can't create backup file, %s", err) continue } gz := gzip.NewWriter(fh) - if err = exporter.Export(gz, ""); err != nil { + if err = exporter.Export(gz, siteID); err != nil { log.Printf("[WARN] export failed, %+v", err) } _ = gz.Close() _ = fh.Close() + + removeOldBackupFiles(backupLocation, siteID, keepMax) + } +} + +func removeOldBackupFiles(backupLocation string, siteID string, keepMax int) { + files, err := ioutil.ReadDir(backupLocation) + if err != nil { + log.Printf("[WARN] can't read files in backup directory %s, %s", backupLocation, err) + return + } + backFiles := []os.FileInfo{} + for _, file := range files { + if strings.HasPrefix(file.Name(), "backup-"+siteID) { + backFiles = append(backFiles, file) + } + } + sort.Slice(backFiles, func(i int, j int) bool { return backFiles[i].Name() < backFiles[j].Name() }) + + if len(backFiles) > keepMax { + for i := 0; i < len(backFiles)-keepMax; i++ { + fpath := backupLocation + "/" + backFiles[i].Name() + if e := os.Remove(fpath); e != nil { + log.Printf("[WARN] can't delete %s, %s", fpath, err) + continue + } + log.Printf("[DEBUG] removed %s", fpath) + } } } diff --git a/app/migrator/migrator_test.go b/app/migrator/migrator_test.go new file mode 100644 index 00000000..8e1db7e7 --- /dev/null +++ b/app/migrator/migrator_test.go @@ -0,0 +1,30 @@ +package migrator + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMigrator_RemoveOldBackupFiles(t *testing.T) { + loc := "tmp/remark-backups.test" + defer os.RemoveAll(loc) + + os.MkdirAll(loc, 0700) + for i := 0; i < 10; i++ { + fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i) + err := ioutil.WriteFile(fname, []byte("blah"), 0600) + assert.Nil(t, err) + } + + removeOldBackupFiles(loc, "site1", 3) + ff, err := ioutil.ReadDir(loc) + assert.Nil(t, err) + assert.Equal(t, 3, len(ff), "should keep 3 files only") + assert.Equal(t, "backup-site1-20171207.gz", ff[0].Name()) + assert.Equal(t, "backup-site1-20171208.gz", ff[1].Name()) + assert.Equal(t, "backup-site1-20171209.gz", ff[2].Name()) +} diff --git a/app/rest/admin.go b/app/rest/admin.go index 5eaa3e1e..ae31ddc6 100644 --- a/app/rest/admin.go +++ b/app/rest/admin.go @@ -36,7 +36,7 @@ func (a *admin) routes() chi.Router { return router } -// DELETE /comment/{id}?site=siteID&url=post-url +// DELETE /comment/{id}?site=siteID&url=post-url - removes comment func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") @@ -54,7 +54,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"id": id, "loc": locator}) } -// PUT /user/{userid}?site=side-id&block=1 +// PUT /user/{userid}?site=side-id&block=1 - block or unblock user func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) { userID := chi.URLParam(r, "userid") siteID := r.URL.Query().Get("site") @@ -68,7 +68,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus}) } -// PUT /pin/{id}?site=siteID&url=post-url&pin=1 +// PUT /pin/{id}?site=siteID&url=post-url&pin=1 - mark/unmark comment as a special func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) { commentID := chi.URLParam(r, "id") locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} @@ -82,7 +82,7 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"id": commentID, "loc": locator, "pin": pinStatus}) } -// GET /export?site=site-id?mode=file|stream +// GET /export?site=site-id?mode=file|stream - exports all comments for siteID as json stream or file func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") var writer io.Writer = w @@ -105,6 +105,7 @@ func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) { if err := a.importer.Import(r.Body, siteID); err != nil { httpError(w, r, http.StatusBadRequest, err, "import failed") } + a.respCache.Flush() } func (a *admin) checkBlocked(locator store.Locator, user store.User) bool { diff --git a/app/rest/server.go b/app/rest/server.go index 0e18ba7f..51a03a39 100644 --- a/app/rest/server.go +++ b/app/rest/server.go @@ -44,7 +44,8 @@ type Server struct { func (s *Server) Run() { log.Print("[INFO] activate rest server") - applyDevMode := func(mode auth.Mode) (modes []auth.Mode) { + // add auth.Developer flag if dev mode is active + maybeDevMode := func(mode auth.Mode) (modes []auth.Mode) { modes = append(modes, mode) if s.DevMode { modes = append(modes, auth.Developer) @@ -57,7 +58,7 @@ func (s *Server) Run() { router := chi.NewRouter() router.Use(middleware.RealIP, Recoverer) router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second)) - router.Use(auth.Auth(s.SessionStore, s.Admins, applyDevMode(auth.Anonymous))) + router.Use(auth.Auth(s.SessionStore, s.Admins, maybeDevMode(auth.Anonymous))) router.Use(Limiter(10), AppInfo("remark", s.Version), Ping, Logger(LogAll)) // If you aren't using gorilla/mux, you need to wrap your handlers with context.ClearHandler @@ -78,7 +79,7 @@ func (s *Server) Run() { rapi.Get("/count", s.countCtrl) // require auth - rapi.With(auth.Auth(s.SessionStore, s.Admins, applyDevMode(auth.Full))).Group(func(rauth chi.Router) { + rapi.With(auth.Auth(s.SessionStore, s.Admins, maybeDevMode(auth.Full))).Group(func(rauth chi.Router) { rauth.Post("/comment", s.createCommentCtrl) rauth.Get("/user", s.userInfoCtrl) rauth.Put("/vote/{id}", s.voteCtrl) @@ -95,26 +96,7 @@ func (s *Server) Run() { log.Fatal(http.ListenAndServe(":8080", router)) } -func (s *Server) addFileServer(r chi.Router, path string, root http.FileSystem) { - fs := http.StripPrefix(path, http.FileServer(root)) - - if path != "/" && path[len(path)-1] != '/' { - r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP) - path += "/" - } - path += "*" - - r.Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // don't show dirs, just serve files - if strings.HasSuffix(r.URL.Path, "/") { - http.NotFound(w, r) - return - } - fs.ServeHTTP(w, r) - })) -} - -// POST /comment +// POST /comment - adds comment, resets all immutable fields func (s *Server) createCommentCtrl(w http.ResponseWriter, r *http.Request) { comment := store.Comment{} @@ -151,7 +133,7 @@ func (s *Server) createCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - s.respCache.Flush() + s.respCache.Flush() // reset all caches render.Status(r, http.StatusAccepted) render.JSON(w, r, JSON{"id": id, "loc": comment.Locator}) @@ -177,7 +159,8 @@ func (s *Server) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"id": id, "loc": locator}) } -// GET /find?site=siteID&url=post-url&format=tree&sort=-time +// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score] +// find comments for given post. Retruns in tree or plain formats, sorted func (s *Server) findCommentsCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} log.Printf("[DEBUG] get comments for %+v", locator) @@ -204,7 +187,7 @@ func (s *Server) findCommentsCtrl(w http.ResponseWriter, r *http.Request) { renderJSONWithHTML(w, r, comments) } -// GET /last/{max}?site=siteID +// GET /last/{max}?site=siteID - last comments for the siteID, across all posts, sorted by time func (s *Server) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) { max, err := strconv.Atoi(chi.URLParam(r, "max")) @@ -230,7 +213,7 @@ func (s *Server) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) { renderJSONWithHTML(w, r, comments) } -// GET /id/{id}?site=siteID&url=post-url +// GET /id/{id}?site=siteID&url=post-url - gets a comment by id func (s *Server) commentByIDCtrl(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") @@ -248,7 +231,7 @@ func (s *Server) commentByIDCtrl(w http.ResponseWriter, r *http.Request) { renderJSONWithHTML(w, r, comment) } -// GET /comments?site=siteID&user=id +// GET /comments?site=siteID&user=id - returns commens for given userID func (s *Server) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) { userID := r.URL.Query().Get("user") @@ -273,7 +256,7 @@ func (s *Server) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) { renderJSONWithHTML(w, r, comments) } -// GET /user +// GET /user - returns user info func (s *Server) userInfoCtrl(w http.ResponseWriter, r *http.Request) { user, err := auth.GetUserInfo(r) if err != nil { @@ -283,7 +266,7 @@ func (s *Server) userInfoCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, user) } -// GET /count?site=siteID&url=post-url +// GET /count?site=siteID&url=post-url - get number of comments for given post func (s *Server) countCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} count, err := s.DataService.Count(locator) @@ -294,7 +277,7 @@ func (s *Server) countCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"count": count, "loc": locator}) } -// PUT /vote/{id}?site=siteID&url=post-url&vote=1 +// PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment func (s *Server) voteCtrl(w http.ResponseWriter, r *http.Request) { user, err := auth.GetUserInfo(r) @@ -338,3 +321,22 @@ func renderJSONWithHTML(w http.ResponseWriter, r *http.Request, v interface{}) { } _, _ = w.Write(buf.Bytes()) } + +func (s *Server) addFileServer(r chi.Router, path string, root http.FileSystem) { + fs := http.StripPrefix(path, http.FileServer(root)) + + if path != "/" && path[len(path)-1] != '/' { + r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP) + path += "/" + } + path += "*" + + r.Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // don't show dirs, just serve files + if strings.HasSuffix(r.URL.Path, "/") { + http.NotFound(w, r) + return + } + fs.ServeHTTP(w, r) + })) +} diff --git a/app/store/bolt.go b/app/store/bolt.go index eea59447..c08fc84e 100644 --- a/app/store/bolt.go +++ b/app/store/bolt.go @@ -86,8 +86,8 @@ func (b *BoltDB) Create(comment Comment) (string, error) { return errors.Wrap(jerr, "can't marshal comment") } - if err := bucket.Put([]byte(comment.ID), jdata); err != nil { - return errors.Wrapf(err, "failed to put key %s", comment.ID) + if e = bucket.Put([]byte(comment.ID), jdata); err != nil { + return errors.Wrapf(e, "failed to put key %s", comment.ID) } // add reference to comment to "last" bucket