mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

auto celanup for backups

Browse Source
This commit is contained in:
Umputun
2017-12-29 18:52:50 -06:00
parent 65d254e586
commit 5d48845a5d
6 changed files with 109 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.go
+5 -1
View File
@@ -26,6 +26,7 @@ var opts struct {
Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"/tmp" description:"backups location"`
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
ServerCommand struct {
SessionStore string `long:"session" env:"SESSION_STORE" default:"/tmp" description:"path to session store directory"`
@@ -112,7 +113,10 @@ func main() {
log.Printf("[WARN] running in dev mode, no auth!")
}
go migrator.AutoBackup(&exporter, opts.BackupLocation)
for _, siteID := range opts.Sites {
go migrator.AutoBackup(&exporter, opts.BackupLocation, siteID, opts.MaxBackupFiles)
}
srv.Run()
}
app/migrator/migrator.go
+34 -3
View File
@@ -4,8 +4,11 @@ import (
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"sort"
"strings"
"time"
"github.com/pkg/errors"
@@ -59,22 +62,50 @@ func ImportComments(p ImportParams) error {
}
// AutoBackup runs daily export to local files
func AutoBackup(exporter Exporter, backupLocation string) {
func AutoBackup(exporter Exporter, backupLocation string, siteID string, keepMax int) {
log.Print("[INFO] activate auto-backup")
tick := time.NewTicker(24 * time.Hour)
for range tick.C {
log.Print("[DEBUG] make backup")
fh, err := os.Create(fmt.Sprintf("%s/backup-%s.gz", backupLocation, time.Now().Format("20060102")))
fh, err := os.Create(fmt.Sprintf("%s/backup-%s-%s.gz", backupLocation, siteID, time.Now().Format("20060102")))
if err != nil {
log.Printf("[WARN] can't create backup file, %s", err)
continue
}
gz := gzip.NewWriter(fh)
if err = exporter.Export(gz, ""); err != nil {
if err = exporter.Export(gz, siteID); err != nil {
log.Printf("[WARN] export failed, %+v", err)
}
_ = gz.Close()
_ = fh.Close()
removeOldBackupFiles(backupLocation, siteID, keepMax)
}
}
func removeOldBackupFiles(backupLocation string, siteID string, keepMax int) {
files, err := ioutil.ReadDir(backupLocation)
if err != nil {
log.Printf("[WARN] can't read files in backup directory %s, %s", backupLocation, err)
return
}
backFiles := []os.FileInfo{}
for _, file := range files {
if strings.HasPrefix(file.Name(), "backup-"+siteID) {
backFiles = append(backFiles, file)
}
}
sort.Slice(backFiles, func(i int, j int) bool { return backFiles[i].Name() < backFiles[j].Name() })
if len(backFiles) > keepMax {
for i := 0; i < len(backFiles)-keepMax; i++ {
fpath := backupLocation + "/" + backFiles[i].Name()
if e := os.Remove(fpath); e != nil {
log.Printf("[WARN] can't delete %s, %s", fpath, err)
continue
}
log.Printf("[DEBUG] removed %s", fpath)
}
}
}
app/migrator/migrator_test.go
+30
View File
@@ -0,0 +1,30 @@
package migrator
import (
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestMigrator_RemoveOldBackupFiles(t *testing.T) {
loc := "tmp/remark-backups.test"
defer os.RemoveAll(loc)
os.MkdirAll(loc, 0700)
for i := 0; i < 10; i++ {
fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
err := ioutil.WriteFile(fname, []byte("blah"), 0600)
assert.Nil(t, err)
}
removeOldBackupFiles(loc, "site1", 3)
ff, err := ioutil.ReadDir(loc)
assert.Nil(t, err)
assert.Equal(t, 3, len(ff), "should keep 3 files only")
assert.Equal(t, "backup-site1-20171207.gz", ff[0].Name())
assert.Equal(t, "backup-site1-20171208.gz", ff[1].Name())
assert.Equal(t, "backup-site1-20171209.gz", ff[2].Name())
}
app/rest/admin.go
+5 -4
View File
@@ -36,7 +36,7 @@ func (a *admin) routes() chi.Router {
return router
}
// DELETE /comment/{id}?site=siteID&url=post-url
// DELETE /comment/{id}?site=siteID&url=post-url - removes comment
func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
@@ -54,7 +54,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, JSON{"id": id, "loc": locator})
}
// PUT /user/{userid}?site=side-id&block=1
// PUT /user/{userid}?site=side-id&block=1 - block or unblock user
func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
userID := chi.URLParam(r, "userid")
siteID := r.URL.Query().Get("site")
@@ -68,7 +68,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
}
// PUT /pin/{id}?site=siteID&url=post-url&pin=1
// PUT /pin/{id}?site=siteID&url=post-url&pin=1 - mark/unmark comment as a special
func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
commentID := chi.URLParam(r, "id")
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
@@ -82,7 +82,7 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, JSON{"id": commentID, "loc": locator, "pin": pinStatus})
}
// GET /export?site=site-id?mode=file|stream
// GET /export?site=site-id?mode=file|stream - exports all comments for siteID as json stream or file
func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
var writer io.Writer = w
@@ -105,6 +105,7 @@ func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) {
if err := a.importer.Import(r.Body, siteID); err != nil {
httpError(w, r, http.StatusBadRequest, err, "import failed")
}
a.respCache.Flush()
}
func (a *admin) checkBlocked(locator store.Locator, user store.User) bool {
app/rest/server.go
+33 -31
View File
@@ -44,7 +44,8 @@ type Server struct {
func (s *Server) Run() {
log.Print("[INFO] activate rest server")
applyDevMode := func(mode auth.Mode) (modes []auth.Mode) {
// add auth.Developer flag if dev mode is active
maybeDevMode := func(mode auth.Mode) (modes []auth.Mode) {
modes = append(modes, mode)
if s.DevMode {
modes = append(modes, auth.Developer)
@@ -57,7 +58,7 @@ func (s *Server) Run() {
router := chi.NewRouter()
router.Use(middleware.RealIP, Recoverer)
router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
router.Use(auth.Auth(s.SessionStore, s.Admins, applyDevMode(auth.Anonymous)))
router.Use(auth.Auth(s.SessionStore, s.Admins, maybeDevMode(auth.Anonymous)))
router.Use(Limiter(10), AppInfo("remark", s.Version), Ping, Logger(LogAll))
// If you aren't using gorilla/mux, you need to wrap your handlers with context.ClearHandler
@@ -78,7 +79,7 @@ func (s *Server) Run() {
rapi.Get("/count", s.countCtrl)
// require auth
rapi.With(auth.Auth(s.SessionStore, s.Admins, applyDevMode(auth.Full))).Group(func(rauth chi.Router) {
rapi.With(auth.Auth(s.SessionStore, s.Admins, maybeDevMode(auth.Full))).Group(func(rauth chi.Router) {
rauth.Post("/comment", s.createCommentCtrl)
rauth.Get("/user", s.userInfoCtrl)
rauth.Put("/vote/{id}", s.voteCtrl)
@@ -95,26 +96,7 @@ func (s *Server) Run() {
log.Fatal(http.ListenAndServe(":8080", router))
}
func (s *Server) addFileServer(r chi.Router, path string, root http.FileSystem) {
fs := http.StripPrefix(path, http.FileServer(root))
if path != "/" && path[len(path)-1] != '/' {
r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
path += "/"
}
path += "*"
r.Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// don't show dirs, just serve files
if strings.HasSuffix(r.URL.Path, "/") {
http.NotFound(w, r)
return
}
fs.ServeHTTP(w, r)
}))
}
// POST /comment
// POST /comment - adds comment, resets all immutable fields
func (s *Server) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
comment := store.Comment{}
@@ -151,7 +133,7 @@ func (s *Server) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
s.respCache.Flush()
s.respCache.Flush() // reset all caches
render.Status(r, http.StatusAccepted)
render.JSON(w, r, JSON{"id": id, "loc": comment.Locator})
@@ -177,7 +159,8 @@ func (s *Server) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, JSON{"id": id, "loc": locator})
}
// GET /find?site=siteID&url=post-url&format=tree&sort=-time
// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
// find comments for given post. Retruns in tree or plain formats, sorted
func (s *Server) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
log.Printf("[DEBUG] get comments for %+v", locator)
@@ -204,7 +187,7 @@ func (s *Server) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
renderJSONWithHTML(w, r, comments)
}
// GET /last/{max}?site=siteID
// GET /last/{max}?site=siteID - last comments for the siteID, across all posts, sorted by time
func (s *Server) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
max, err := strconv.Atoi(chi.URLParam(r, "max"))
@@ -230,7 +213,7 @@ func (s *Server) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
renderJSONWithHTML(w, r, comments)
}
// GET /id/{id}?site=siteID&url=post-url
// GET /id/{id}?site=siteID&url=post-url - gets a comment by id
func (s *Server) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
@@ -248,7 +231,7 @@ func (s *Server) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
renderJSONWithHTML(w, r, comment)
}
// GET /comments?site=siteID&user=id
// GET /comments?site=siteID&user=id - returns commens for given userID
func (s *Server) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
userID := r.URL.Query().Get("user")
@@ -273,7 +256,7 @@ func (s *Server) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
renderJSONWithHTML(w, r, comments)
}
// GET /user
// GET /user - returns user info
func (s *Server) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
user, err := auth.GetUserInfo(r)
if err != nil {
@@ -283,7 +266,7 @@ func (s *Server) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, user)
}
// GET /count?site=siteID&url=post-url
// GET /count?site=siteID&url=post-url - get number of comments for given post
func (s *Server) countCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
count, err := s.DataService.Count(locator)
@@ -294,7 +277,7 @@ func (s *Server) countCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, JSON{"count": count, "loc": locator})
}
// PUT /vote/{id}?site=siteID&url=post-url&vote=1
// PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
func (s *Server) voteCtrl(w http.ResponseWriter, r *http.Request) {
user, err := auth.GetUserInfo(r)
@@ -338,3 +321,22 @@ func renderJSONWithHTML(w http.ResponseWriter, r *http.Request, v interface{}) {
}
_, _ = w.Write(buf.Bytes())
}
func (s *Server) addFileServer(r chi.Router, path string, root http.FileSystem) {
fs := http.StripPrefix(path, http.FileServer(root))
if path != "/" && path[len(path)-1] != '/' {
r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
path += "/"
}
path += "*"
r.Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// don't show dirs, just serve files
if strings.HasSuffix(r.URL.Path, "/") {
http.NotFound(w, r)
return
}
fs.ServeHTTP(w, r)
}))
}
app/store/bolt.go
+2 -2
View File
@@ -86,8 +86,8 @@ func (b *BoltDB) Create(comment Comment) (string, error) {
return errors.Wrap(jerr, "can't marshal comment")
}
if err := bucket.Put([]byte(comment.ID), jdata); err != nil {
return errors.Wrapf(err, "failed to put key %s", comment.ID)
if e = bucket.Put([]byte(comment.ID), jdata); err != nil {
return errors.Wrapf(e, "failed to put key %s", comment.ID)
}
// add reference to comment to "last" bucket