From bb4a54ff71f93c9a1af6407338194c51612bbd38 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 20:44:02 -0500 Subject: [PATCH 01/15] main split to setup and run stages, with graceful shutdown --- app/main.go | 92 ++++++++++++++++++++++++++++++------------ app/migrator/backup.go | 22 ++++++---- app/rest/api/import.go | 15 ++++++- app/rest/api/rest.go | 9 +++++ 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/app/main.go b/app/main.go index 3b86d3d9..152cbe8b 100644 --- a/app/main.go +++ b/app/main.go @@ -1,11 +1,14 @@ package main import ( + "context" "fmt" "log" "net/http" "os" + "os/signal" "strings" + "syscall" "time" "github.com/coreos/bbolt" @@ -22,7 +25,7 @@ import ( "github.com/umputun/remark/app/rest/proxy" ) -var opts struct { +type Opts struct { BoltPath string `long:"bolt" env:"BOLTDB_PATH" default:"./var" description:"parent dir for bolt files"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` RemarkURL string `long:"url" env:"REMARK_URL" default:"https://remark42.com" description:"url to remark"` @@ -56,49 +59,80 @@ var opts struct { WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` } +var opts Opts var revision = "unknown" +type application struct { + Opts + srv api.Rest + importer api.Import + exporter migrator.Exporter +} + func main() { fmt.Printf("remark %s\n", revision) p := flags.NewParser(&opts, flags.Default) if _, e := p.ParseArgs(os.Args[1:]); e != nil { os.Exit(1) } - - setupLog(opts.Dbg) log.Print("[INFO] started remark") - if err := makeDirs(opts.BoltPath, opts.BackupLocation, opts.AvatarStore); err != nil { - log.Fatalf("[ERROR] can't create directories, %+v", err) + ctx, cancel := context.WithCancel(context.Background()) + + // graceful termination + go func() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop + log.Print("[WARN] interrupt signal") + cancel() + }() + + app, err := Setup(opts) + if err != nil { + log.Fatalf("[ERROR] failed to setup application, %+v", err) } + log.Printf("[INFO] remark terminated %s", Run(ctx, app)) +} - dataStore := makeBoltStore(opts.Sites) - - if opts.DevPasswd != "" { +// Run all application objects +func Run(ctx context.Context, a *application) error { + if a.DevPasswd != "" { log.Printf("[WARN] running in dev mode") } + activateBackup(ctx, a.exporter) // runs in goroutine for each site + go a.importer.Run(a.Port + 1) + go a.srv.Run(opts.Port) + + // shutdown on context cancellation + <-ctx.Done() + a.srv.Shutdown() + a.importer.Shutdown() + + return ctx.Err() +} + +// Setup prepares application and return all active parts +// doesn't start anything +func Setup(opts Opts) (*application, error) { + setupLog(opts.Dbg) + + if err := makeDirs(opts.BoltPath, opts.BackupLocation, opts.AvatarStore); err != nil { + return nil, err + } + dataService := service.DataStore{ - Interface: dataStore, + Interface: makeBoltStore(opts.Sites), EditDuration: 5 * time.Minute, Secret: opts.SecretKey, MaxCommentSize: opts.MaxCommentSize, } - exporter := migrator.Remark{DataStore: &dataService} cache := rest.NewLoadingCache(rest.MaxValueSize(opts.MaxCachedValue), rest.MaxKeys(opts.MaxCachedItems), rest.PostFlushFn(postFlushFn)) - activateBackup(&exporter) - - importSrv := api.Import{ - Version: revision, - Cache: cache, - NativeImporter: &migrator.Remark{DataStore: &dataService}, - DisqusImporter: &migrator.Disqus{DataStore: &dataService}, - SecretKey: opts.SecretKey, - } - go importSrv.Run(opts.Port + 1) + jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour) avatarProxy := &proxy.Avatar{ StorePath: opts.AvatarStore, @@ -106,12 +140,20 @@ func main() { RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), } - jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour) + exporter := &migrator.Remark{DataStore: &dataService} + + importer := api.Import{ + Version: revision, + Cache: cache, + NativeImporter: &migrator.Remark{DataStore: &dataService}, + DisqusImporter: &migrator.Disqus{DataStore: &dataService}, + SecretKey: opts.SecretKey, + } srv := api.Rest{ Version: revision, DataService: dataService, - Exporter: &exporter, + Exporter: exporter, WebRoot: opts.WebRoot, ImageProxy: proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}, Authenticator: auth.Authenticator{ @@ -124,11 +166,11 @@ func main() { Cache: cache, } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore - srv.Run(opts.Port) + return &application{srv: srv, importer: importer, exporter: exporter, Opts: opts}, nil } // activateBackup runs background backups for each site -func activateBackup(exporter migrator.Exporter) { +func activateBackup(ctx context.Context, exporter migrator.Exporter) { for _, siteID := range opts.Sites { backup := migrator.AutoBackup{ Exporter: exporter, @@ -137,7 +179,7 @@ func activateBackup(exporter migrator.Exporter) { KeepMax: opts.MaxBackupFiles, Duration: 24 * time.Hour, } - go backup.Do() + go backup.Do(ctx) } } diff --git a/app/migrator/backup.go b/app/migrator/backup.go index 155ef75c..c0fcff59 100644 --- a/app/migrator/backup.go +++ b/app/migrator/backup.go @@ -2,6 +2,7 @@ package migrator import ( "compress/gzip" + "context" "fmt" "io/ioutil" "log" @@ -23,17 +24,24 @@ type AutoBackup struct { } // Do runs daily export to local files, keeps up to keepMax backups for given siteID -func (ab AutoBackup) Do() { +func (ab AutoBackup) Do(ctx context.Context) { log.Printf("[INFO] activate auto-backup for %s", ab.BackupLocation) tick := time.NewTicker(ab.Duration) log.Printf("[DEBUG] first backup at %s", time.Now().Add(ab.Duration)) - for range tick.C { - if _, err := ab.makeBackup(); err != nil { - log.Printf("[WARN] auto-backup for %s failed, %s", ab.SiteID, err) - continue + + for { + select { + case <-tick.C: + if _, err := ab.makeBackup(); err != nil { + log.Printf("[WARN] auto-backup for %s failed, %s", ab.SiteID, err) + continue + } + ab.removeOldBackupFiles() + log.Printf("[DEBUG] next backup at %s", time.Now().Add(ab.Duration)) + case <-ctx.Done(): + log.Printf("[WARN] terminated autobackup for %s", ab.SiteID) + return } - ab.removeOldBackupFiles() - log.Printf("[DEBUG] next backup at %s", time.Now().Add(ab.Duration)) } } diff --git a/app/rest/api/import.go b/app/rest/api/import.go index 2f34da6d..a69d18f9 100644 --- a/app/rest/api/import.go +++ b/app/rest/api/import.go @@ -1,6 +1,7 @@ package api import ( + "context" "fmt" "log" "net/http" @@ -24,6 +25,8 @@ type Import struct { NativeImporter migrator.Importer DisqusImporter migrator.Importer SecretKey string + + httpServer *http.Server } // Run the listener and request's router, activate rest server @@ -31,11 +34,19 @@ type Import struct { func (s *Import) Run(port int) { log.Printf("[INFO] activate import server on port %d", port) router := s.routes() - httpServer := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router} - err := httpServer.ListenAndServe() + s.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router} + err := s.httpServer.ListenAndServe() log.Printf("[WARN] http server terminated, %s", err) } +func (s *Import) Shutdown() { + log.Print("[WARN] shutdown import server") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + s.httpServer.Shutdown(ctx) + log.Print("[DEBUG] shutdown import server completed") +} + func (s Import) routes() chi.Router { router := chi.NewRouter() router.Use(middleware.RealIP, Recoverer) diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 717fec68..a36f9e8c 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -2,6 +2,7 @@ package api import ( "bytes" + "context" "crypto/sha1" "encoding/base64" "encoding/json" @@ -74,6 +75,14 @@ func (s *Rest) Run(port int) { log.Printf("[WARN] http server terminated, %s", err) } +func (s *Rest) Shutdown() { + log.Print("[WARN] shutdown rest server") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + s.httpServer.Shutdown(ctx) + log.Print("[DEBUG] shutdown rest server completed") +} + func (s *Rest) routes() chi.Router { router := chi.NewRouter() router.Use(middleware.RealIP, Recoverer) From d7d90eeb4e939f90f33ee0b7d1713126fa7e914b Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 20:56:52 -0500 Subject: [PATCH 02/15] lint: bunch of minor warnings --- app/main.go | 10 ++++++---- app/rest/api/import.go | 5 ++++- app/rest/api/import_test.go | 1 + app/rest/api/rest.go | 5 ++++- app/rest/auth/auth_test.go | 8 ++++++++ app/rest/auth/provider_test.go | 1 + app/rest/cache_test.go | 1 + app/rest/proxy/image_test.go | 4 ++-- 8 files changed, 27 insertions(+), 8 deletions(-) diff --git a/app/main.go b/app/main.go index 152cbe8b..0e18510a 100644 --- a/app/main.go +++ b/app/main.go @@ -25,6 +25,7 @@ import ( "github.com/umputun/remark/app/rest/proxy" ) +// Opts with command line flags and env type Opts struct { BoltPath string `long:"bolt" env:"BOLTDB_PATH" default:"./var" description:"parent dir for bolt files"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` @@ -62,7 +63,8 @@ type Opts struct { var opts Opts var revision = "unknown" -type application struct { +// Application holds all active objects +type Application struct { Opts srv api.Rest importer api.Import @@ -96,7 +98,7 @@ func main() { } // Run all application objects -func Run(ctx context.Context, a *application) error { +func Run(ctx context.Context, a *Application) error { if a.DevPasswd != "" { log.Printf("[WARN] running in dev mode") } @@ -115,7 +117,7 @@ func Run(ctx context.Context, a *application) error { // Setup prepares application and return all active parts // doesn't start anything -func Setup(opts Opts) (*application, error) { +func Setup(opts Opts) (*Application, error) { setupLog(opts.Dbg) if err := makeDirs(opts.BoltPath, opts.BackupLocation, opts.AvatarStore); err != nil { @@ -166,7 +168,7 @@ func Setup(opts Opts) (*application, error) { Cache: cache, } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore - return &application{srv: srv, importer: importer, exporter: exporter, Opts: opts}, nil + return &Application{srv: srv, importer: importer, exporter: exporter, Opts: opts}, nil } // activateBackup runs background backups for each site diff --git a/app/rest/api/import.go b/app/rest/api/import.go index a69d18f9..c7318d3e 100644 --- a/app/rest/api/import.go +++ b/app/rest/api/import.go @@ -39,11 +39,14 @@ func (s *Import) Run(port int) { log.Printf("[WARN] http server terminated, %s", err) } +// Shutdown import http server func (s *Import) Shutdown() { log.Print("[WARN] shutdown import server") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - s.httpServer.Shutdown(ctx) + if err := s.httpServer.Shutdown(ctx); err != nil { + log.Printf("[DEBUG] importer shutdown error, %s", err) + } log.Print("[DEBUG] shutdown import server completed") } diff --git a/app/rest/api/import_test.go b/app/rest/api/import_test.go index 6d6e06e4..8fd59994 100644 --- a/app/rest/api/import_test.go +++ b/app/rest/api/import_test.go @@ -30,6 +30,7 @@ func TestImport(t *testing.T) { req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r) assert.Nil(t, err) resp, err := client.Do(req) + assert.Nil(t, err) assert.Equal(t, http.StatusCreated, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index a36f9e8c..23356638 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -75,11 +75,14 @@ func (s *Rest) Run(port int) { log.Printf("[WARN] http server terminated, %s", err) } +// Shutdown rest http server func (s *Rest) Shutdown() { log.Print("[WARN] shutdown rest server") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - s.httpServer.Shutdown(ctx) + if err := s.httpServer.Shutdown(ctx); err != nil { + log.Printf("[DEBUG] rest shutdown error, %s", err) + } log.Print("[DEBUG] shutdown rest server completed") } diff --git a/app/rest/auth/auth_test.go b/app/rest/auth/auth_test.go index b6df5ac2..ddfc2f92 100644 --- a/app/rest/auth/auth_test.go +++ b/app/rest/auth/auth_test.go @@ -87,17 +87,20 @@ func TestAuthRequired(t *testing.T) { client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) req = withBasicAuth(req, "dev", "123456") resp, err := client.Do(req) require.NoError(t, err) assert.Equal(t, 201, resp.StatusCode, "valid auth user") req, err = http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) resp, err = client.Do(req) require.NoError(t, err) assert.Equal(t, 401, resp.StatusCode, "no auth user") req, err = http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) req = withBasicAuth(req, "dev", "xyz") resp, err = client.Do(req) require.NoError(t, err) @@ -115,17 +118,20 @@ func TestAuthNotRequired(t *testing.T) { client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) req = withBasicAuth(req, "dev", "123456") resp, err := client.Do(req) require.NoError(t, err) assert.Equal(t, 201, resp.StatusCode, "valid auth user") req, err = http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) resp, err = client.Do(req) require.NoError(t, err) assert.Equal(t, 201, resp.StatusCode, "no auth user") req, err = http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) req = withBasicAuth(req, "dev", "ZZZZ123456") resp, err = client.Do(req) require.NoError(t, err) @@ -143,6 +149,7 @@ func TestAdminRequired(t *testing.T) { client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) req = withBasicAuth(req, "dev", "123456") resp, err := client.Do(req) require.NoError(t, err) @@ -150,6 +157,7 @@ func TestAdminRequired(t *testing.T) { devUser.Admin = false req, err = http.NewRequest("GET", server.URL+"/auth", nil) + require.NoError(t, err) req = withBasicAuth(req, "dev", "123456") resp, err = client.Do(req) require.NoError(t, err) diff --git a/app/rest/auth/provider_test.go b/app/rest/auth/provider_test.go index e6f6dcd6..f238043b 100644 --- a/app/rest/auth/provider_test.go +++ b/app/rest/auth/provider_test.go @@ -27,6 +27,7 @@ func TestLogin(t *testing.T) { }() jar, err := cookiejar.New(nil) + require.Nil(t, err) client := &http.Client{Jar: jar, Timeout: 5 * time.Second} resp, err := client.Get("http://localhost:8981/login") assert.Nil(t, err) diff --git a/app/rest/cache_test.go b/app/rest/cache_test.go index 645feb47..cd057dc3 100644 --- a/app/rest/cache_test.go +++ b/app/rest/cache_test.go @@ -157,6 +157,7 @@ func TestLoadingCache_Parallel(t *testing.T) { wg := sync.WaitGroup{} for i := 0; i < 1000; i++ { wg.Add(1) + i := i go func() { defer wg.Done() res, err := lc.Get("key", time.Minute, func() ([]byte, error) { diff --git a/app/rest/proxy/image_test.go b/app/rest/proxy/image_test.go index 31d48498..28f444be 100644 --- a/app/rest/proxy/image_test.go +++ b/app/rest/proxy/image_test.go @@ -62,7 +62,7 @@ func TestImage_Routes(t *testing.T) { img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"} router := img.Routes() - httpSrv := imgHttpServer(t) + httpSrv := imgHTTPServer(t) defer httpSrv.Close() ts := httptest.NewServer(router) defer ts.Close() @@ -88,7 +88,7 @@ func TestPicture_Convert(t *testing.T) { assert.Equal(t, ` xyz `, r) } -func imgHttpServer(t *testing.T) *httptest.Server { +func imgHTTPServer(t *testing.T) *httptest.Server { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/image/img1.png" { t.Log("http img request", r.URL) From e3f8e6d2dcc525b53ca208ddc2e6b0d55588ec3e Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 21:04:50 -0500 Subject: [PATCH 03/15] bolt test fixed --- app/store/engine/bolt_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/store/engine/bolt_test.go b/app/store/engine/bolt_test.go index 5a8edc9a..1f0ac59e 100644 --- a/app/store/engine/bolt_test.go +++ b/app/store/engine/bolt_test.go @@ -14,8 +14,8 @@ import ( var testDb = "/tmp/test-remark.db" func TestBoltDB_CreateAndFind(t *testing.T) { - var b = prep(t) defer os.Remove(testDb) + var b = prep(t) res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") assert.Nil(t, err) @@ -30,8 +30,8 @@ func TestBoltDB_CreateAndFind(t *testing.T) { } func TestBoltDB_Delete(t *testing.T) { - defer os.Remove(testDb) b := prep(t) + defer os.Remove(testDb) loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} res, err := b.Find(loc, "time") @@ -218,7 +218,7 @@ func TestBoltDB_GetForUser(t *testing.T) { func prep(t *testing.T) *BoltDB { os.Remove(testDb) - boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"}) + boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"}) assert.Nil(t, err) b := boltStore From 3883d6414b1b9ef57393ba7b42ed6855b7a4627f Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 21:32:39 -0500 Subject: [PATCH 04/15] make application methods as a part of struct --- app/main.go | 63 +++++++++++++++++++++--------------------- app/rest/cache.go | 4 +-- app/rest/cache_test.go | 4 +-- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/app/main.go b/app/main.go index 0e18510a..8b89800d 100644 --- a/app/main.go +++ b/app/main.go @@ -81,7 +81,7 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) - // graceful termination + // catch signal and invoke graceful termination go func() { stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) @@ -90,34 +90,16 @@ func main() { cancel() }() - app, err := Setup(opts) + app, err := New(opts) if err != nil { log.Fatalf("[ERROR] failed to setup application, %+v", err) } - log.Printf("[INFO] remark terminated %s", Run(ctx, app)) + log.Printf("[INFO] remark terminated %s", app.Run(ctx)) } -// Run all application objects -func Run(ctx context.Context, a *Application) error { - if a.DevPasswd != "" { - log.Printf("[WARN] running in dev mode") - } - - activateBackup(ctx, a.exporter) // runs in goroutine for each site - go a.importer.Run(a.Port + 1) - go a.srv.Run(opts.Port) - - // shutdown on context cancellation - <-ctx.Done() - a.srv.Shutdown() - a.importer.Shutdown() - - return ctx.Err() -} - -// Setup prepares application and return all active parts +// New prepares application and return it with all active parts // doesn't start anything -func Setup(opts Opts) (*Application, error) { +func New(opts Opts) (*Application, error) { setupLog(opts.Dbg) if err := makeDirs(opts.BoltPath, opts.BackupLocation, opts.AvatarStore); err != nil { @@ -125,13 +107,13 @@ func Setup(opts Opts) (*Application, error) { } dataService := service.DataStore{ - Interface: makeBoltStore(opts.Sites), + Interface: makeBoltStore(opts.Sites, opts.BoltPath), EditDuration: 5 * time.Minute, Secret: opts.SecretKey, MaxCommentSize: opts.MaxCommentSize, } - cache := rest.NewLoadingCache(rest.MaxValueSize(opts.MaxCachedValue), rest.MaxKeys(opts.MaxCachedItems), + cache := rest.NewLoadingCache(rest.MaxValSize(opts.MaxCachedValue), rest.MaxKeys(opts.MaxCachedItems), rest.PostFlushFn(postFlushFn)) jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour) @@ -171,14 +153,31 @@ func Setup(opts Opts) (*Application, error) { return &Application{srv: srv, importer: importer, exporter: exporter, Opts: opts}, nil } +// Run all application objects +func (a *Application) Run(ctx context.Context) error { + if a.DevPasswd != "" { + log.Printf("[WARN] running in dev mode") + } + + a.activateBackup(ctx) // runs in goroutine for each site + go a.importer.Run(a.Port + 1) + go a.srv.Run(opts.Port) + + // shutdown on context cancellation + <-ctx.Done() + a.srv.Shutdown() + a.importer.Shutdown() + return ctx.Err() +} + // activateBackup runs background backups for each site -func activateBackup(ctx context.Context, exporter migrator.Exporter) { - for _, siteID := range opts.Sites { +func (a *Application) activateBackup(ctx context.Context) { + for _, siteID := range a.Sites { backup := migrator.AutoBackup{ - Exporter: exporter, - BackupLocation: opts.BackupLocation, + Exporter: a.exporter, + BackupLocation: a.BackupLocation, SiteID: siteID, - KeepMax: opts.MaxBackupFiles, + KeepMax: a.MaxBackupFiles, Duration: 24 * time.Hour, } go backup.Do(ctx) @@ -186,10 +185,10 @@ func activateBackup(ctx context.Context, exporter migrator.Exporter) { } // makeBoltStore creates store for all sites -func makeBoltStore(siteNames []string) engine.Interface { +func makeBoltStore(siteNames []string, path string) engine.Interface { sites := []engine.BoltSite{} for _, site := range siteNames { - sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", opts.BoltPath, site)}) + sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", path, site)}) } result, err := engine.NewBoltDB(bolt.Options{Timeout: 30 * time.Second}, sites...) if err != nil { diff --git a/app/rest/cache.go b/app/rest/cache.go index e4972a38..294612f4 100644 --- a/app/rest/cache.go +++ b/app/rest/cache.go @@ -82,9 +82,9 @@ func (lc *loadingCache) allowed(data []byte) bool { // CacheOption func type type CacheOption func(lc *loadingCache) error -// MaxValueSize functional option defines the largest value's size allowed to be cached +// MaxValSize functional option defines the largest value's size allowed to be cached // By default it is 0, which means unlimited. -func MaxValueSize(max int) CacheOption { +func MaxValSize(max int) CacheOption { return func(lc *loadingCache) error { lc.maxValueSize = max return nil diff --git a/app/rest/cache_test.go b/app/rest/cache_test.go index cd057dc3..2e36ec53 100644 --- a/app/rest/cache_test.go +++ b/app/rest/cache_test.go @@ -43,7 +43,7 @@ func TestLoadingCache_Get(t *testing.T) { func TestLoadingCache_MaxKeys(t *testing.T) { var postFnCall, coldCalls int32 lc := NewLoadingCache(CleanupInterval(200*time.Millisecond), PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }), - MaxKeys(5), MaxValueSize(10)) + MaxKeys(5), MaxValSize(10)) // put 5 keys to cache for i := 0; i < 5; i++ { @@ -97,7 +97,7 @@ func TestLoadingCache_MaxKeys(t *testing.T) { } func TestLoadingCache_MaxSize(t *testing.T) { - lc := NewLoadingCache(CleanupInterval(200*time.Millisecond), MaxKeys(5), MaxValueSize(10)) + lc := NewLoadingCache(CleanupInterval(200*time.Millisecond), MaxKeys(5), MaxValSize(10)) // put good size value to cache and make sure it cached res, err := lc.Get("key-Z", time.Minute, func() ([]byte, error) { From dab4eeb33f06affb7a6b71eab0366a1368c6953c Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 22:46:01 -0500 Subject: [PATCH 05/15] add integration tests --- app/main.go | 81 ++++++++++++++++++++++++------------------ app/main_test.go | 62 ++++++++++++++++++++++++++++++++ app/rest/api/import.go | 10 ++++++ app/rest/api/rest.go | 10 +++++- 4 files changed, 127 insertions(+), 36 deletions(-) create mode 100644 app/main_test.go diff --git a/app/main.go b/app/main.go index 8b89800d..64e4eb79 100644 --- a/app/main.go +++ b/app/main.go @@ -60,19 +60,21 @@ type Opts struct { WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` } -var opts Opts var revision = "unknown" // Application holds all active objects type Application struct { Opts - srv api.Rest - importer api.Import - exporter migrator.Exporter + srv *api.Rest + importer *api.Import + exporter migrator.Exporter + terminated chan struct{} } func main() { fmt.Printf("remark %s\n", revision) + + var opts Opts p := flags.NewParser(&opts, flags.Default) if _, e := p.ParseArgs(os.Args[1:]); e != nil { os.Exit(1) @@ -80,9 +82,7 @@ func main() { log.Print("[INFO] started remark") ctx, cancel := context.WithCancel(context.Background()) - - // catch signal and invoke graceful termination - go func() { + go func() { // catch signal and invoke graceful termination stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) <-stop @@ -94,7 +94,8 @@ func main() { if err != nil { log.Fatalf("[ERROR] failed to setup application, %+v", err) } - log.Printf("[INFO] remark terminated %s", app.Run(ctx)) + err = app.Run(ctx) + log.Printf("[INFO] remark terminated %s", err) } // New prepares application and return it with all active parts @@ -114,7 +115,7 @@ func New(opts Opts) (*Application, error) { } cache := rest.NewLoadingCache(rest.MaxValSize(opts.MaxCachedValue), rest.MaxKeys(opts.MaxCachedItems), - rest.PostFlushFn(postFlushFn)) + rest.PostFlushFn(postFlushFn(opts.Sites, opts.Port))) jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour) @@ -126,7 +127,7 @@ func New(opts Opts) (*Application, error) { exporter := &migrator.Remark{DataStore: &dataService} - importer := api.Import{ + importer := &api.Import{ Version: revision, Cache: cache, NativeImporter: &migrator.Remark{DataStore: &dataService}, @@ -134,7 +135,7 @@ func New(opts Opts) (*Application, error) { SecretKey: opts.SecretKey, } - srv := api.Rest{ + srv := &api.Rest{ Version: revision, DataService: dataService, Exporter: exporter, @@ -143,14 +144,15 @@ func New(opts Opts) (*Application, error) { Authenticator: auth.Authenticator{ JWTService: jwtService, Admins: opts.Admins, - Providers: makeAuthProviders(jwtService, avatarProxy), + Providers: makeAuthProviders(jwtService, avatarProxy, opts), AvatarProxy: avatarProxy, DevPasswd: opts.DevPasswd, }, Cache: cache, } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore - return &Application{srv: srv, importer: importer, exporter: exporter, Opts: opts}, nil + tch := make(chan struct{}) + return &Application{srv: srv, importer: importer, exporter: exporter, Opts: opts, terminated: tch}, nil } // Run all application objects @@ -159,15 +161,22 @@ func (a *Application) Run(ctx context.Context) error { log.Printf("[WARN] running in dev mode") } + go func() { + // shutdown on context cancellation + <-ctx.Done() + a.srv.Shutdown() + a.importer.Shutdown() + }() a.activateBackup(ctx) // runs in goroutine for each site go a.importer.Run(a.Port + 1) - go a.srv.Run(opts.Port) + a.srv.Run(a.Port) + close(a.terminated) + return nil +} - // shutdown on context cancellation - <-ctx.Done() - a.srv.Shutdown() - a.importer.Shutdown() - return ctx.Err() +// Wait for application completion (termination) +func (a *Application) Wait() { + <-a.terminated } // activateBackup runs background backups for each site @@ -226,7 +235,7 @@ func makeDirs(dirs ...string) error { return nil } -func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar) (providers []auth.Provider) { +func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, opts Opts) (providers []auth.Provider) { makeParams := func(cid, secret string) auth.Params { return auth.Params{ @@ -257,23 +266,25 @@ func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar) (provide } // post-flush callback invoked by cache after each flush in async way -func postFlushFn() { +func postFlushFn(sites []string, port int) func() { - // list of heavy urls for pre-heating on cache change - urls := []string{ - "http://localhost:%d/api/v1/list?site=%s", - "http://localhost:%d/api/v1/last/50?site=%s", - } + return func() { + // list of heavy urls for pre-heating on cache change + urls := []string{ + "http://localhost:%d/api/v1/list?site=%s", + "http://localhost:%d/api/v1/last/50?site=%s", + } - for _, site := range opts.Sites { - for _, u := range urls { - resp, err := http.Get(fmt.Sprintf(u, opts.Port, site)) - if err != nil { - log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err) - return - } - if err = resp.Body.Close(); err != nil { - log.Printf("[WARN] failed to close response body, %s", err) + for _, site := range sites { + for _, u := range urls { + resp, err := http.Get(fmt.Sprintf(u, port, site)) + if err != nil { + log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err) + return + } + if err = resp.Body.Close(); err != nil { + log.Printf("[WARN] failed to close response body, %s", err) + } } } } diff --git a/app/main_test.go b/app/main_test.go new file mode 100644 index 00000000..85615b0e --- /dev/null +++ b/app/main_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "net/http" + "testing" + "time" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplication(t *testing.T) { + app, ctx := prepApp(t, 18080, 500*time.Millisecond) + go app.Run(ctx) + time.Sleep(100 * time.Millisecond) // let server start + + resp, err := http.Get("http://localhost:18080/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + app.Wait() +} + +func TestApplicationShutdown(t *testing.T) { + app, ctx := prepApp(t, 18090, 500*time.Millisecond) + st := time.Now() + app.Run(ctx) + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") + app.Wait() +} + +func prepApp(t *testing.T, port int, duration time.Duration) (*Application, context.Context) { + // prepare options + opts := Opts{} + p := flags.NewParser(&opts, flags.Default) + p.ParseArgs([]string{"--secret=123456"}) + opts.AvatarStore, opts.BackupLocation = "/tmp", "/tmp" + opts.BoltPath = fmt.Sprintf("/tmp/%d", port) + opts.GithubCSEC, opts.GithubCID = "csec", "cid" + opts.Port = port + + // create app + app, err := New(opts) + require.Nil(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(duration) + log.Print("[TEST] terminate app") + cancel() + }() + return app, ctx +} diff --git a/app/rest/api/import.go b/app/rest/api/import.go index c7318d3e..e04429bc 100644 --- a/app/rest/api/import.go +++ b/app/rest/api/import.go @@ -6,6 +6,7 @@ import ( "log" "net/http" "strings" + "sync" "time" "github.com/didip/tollbooth" @@ -27,6 +28,7 @@ type Import struct { SecretKey string httpServer *http.Server + lock sync.Mutex } // Run the listener and request's router, activate rest server @@ -34,7 +36,11 @@ type Import struct { func (s *Import) Run(port int) { log.Printf("[INFO] activate import server on port %d", port) router := s.routes() + + s.lock.Lock() s.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router} + s.lock.Unlock() + err := s.httpServer.ListenAndServe() log.Printf("[WARN] http server terminated, %s", err) } @@ -44,9 +50,13 @@ func (s *Import) Shutdown() { log.Print("[WARN] shutdown import server") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() + + s.lock.Lock() if err := s.httpServer.Shutdown(ctx); err != nil { log.Printf("[DEBUG] importer shutdown error, %s", err) } + s.lock.Unlock() + log.Print("[DEBUG] shutdown import server completed") } diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 23356638..21242e2f 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -11,6 +11,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/didip/tollbooth" @@ -44,7 +45,9 @@ type Rest struct { Critical int } - httpServer *http.Server + httpServer *http.Server + lock sync.Mutex + adminService admin } @@ -64,6 +67,7 @@ func (s *Rest) Run(port int) { router := s.routes() + s.lock.Lock() s.httpServer = &http.Server{ Addr: fmt.Sprintf(":%d", port), Handler: router, @@ -71,6 +75,8 @@ func (s *Rest) Run(port int) { WriteTimeout: 5 * time.Second, IdleTimeout: 30 * time.Second, } + s.lock.Unlock() + err := s.httpServer.ListenAndServe() log.Printf("[WARN] http server terminated, %s", err) } @@ -80,10 +86,12 @@ func (s *Rest) Shutdown() { log.Print("[WARN] shutdown rest server") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() + s.lock.Lock() if err := s.httpServer.Shutdown(ctx); err != nil { log.Printf("[DEBUG] rest shutdown error, %s", err) } log.Print("[DEBUG] shutdown rest server completed") + s.lock.Unlock() } func (s *Rest) routes() chi.Router { From f1b22465b0ae53c8d62e55adca70bcbe1298beaf Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 22:50:14 -0500 Subject: [PATCH 06/15] lint: import with value receiver --- app/rest/api/import.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/rest/api/import.go b/app/rest/api/import.go index e04429bc..551b7ef2 100644 --- a/app/rest/api/import.go +++ b/app/rest/api/import.go @@ -60,7 +60,7 @@ func (s *Import) Shutdown() { log.Print("[DEBUG] shutdown import server completed") } -func (s Import) routes() chi.Router { +func (s *Import) routes() chi.Router { router := chi.NewRouter() router.Use(middleware.RealIP, Recoverer) router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second)) From b0e2ff22c340e92b5657eb9270ee4aaed3d4c311 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 23:16:17 -0500 Subject: [PATCH 07/15] test for main terminated with signal --- app/main_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/main_test.go b/app/main_test.go index 85615b0e..75d3e6a9 100644 --- a/app/main_test.go +++ b/app/main_test.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "log" "net/http" + "os" + "syscall" "testing" "time" @@ -38,6 +40,17 @@ func TestApplicationShutdown(t *testing.T) { app.Wait() } +func TestApplicationMainSignal(t *testing.T) { + os.Args = []string{"test", "--secret=123456", "--bolt=/tmp/xyz", "--backup=/tmp", "--avatars=/tmp", "--port=18100"} + go func() { + time.Sleep(100 * time.Millisecond) + syscall.Kill(syscall.Getpid(), syscall.SIGTERM) + }() + st := time.Now() + main() + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") +} + func prepApp(t *testing.T, port int, duration time.Duration) (*Application, context.Context) { // prepare options opts := Opts{} From 60769b5cfc9753a63a55e0b8c9267d5f8c99e4cb Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 23:39:08 -0500 Subject: [PATCH 08/15] add comment to integration test --- app/main_test.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/main_test.go b/app/main_test.go index 75d3e6a9..287fb10b 100644 --- a/app/main_test.go +++ b/app/main_test.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + "strings" "syscall" "testing" "time" @@ -21,14 +22,22 @@ func TestApplication(t *testing.T) { go app.Run(ctx) time.Sleep(100 * time.Millisecond) // let server start + // send ping resp, err := http.Get("http://localhost:18080/api/v1/ping") require.Nil(t, err) defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) body, err := ioutil.ReadAll(resp.Body) assert.Nil(t, err) assert.Equal(t, "pong", string(body)) + + // add comment + resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json", + strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`)) + require.Nil(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + body, _ = ioutil.ReadAll(resp.Body) + t.Log(string(body)) app.Wait() } @@ -55,12 +64,16 @@ func prepApp(t *testing.T, port int, duration time.Duration) (*Application, cont // prepare options opts := Opts{} p := flags.NewParser(&opts, flags.Default) - p.ParseArgs([]string{"--secret=123456"}) + p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password"}) opts.AvatarStore, opts.BackupLocation = "/tmp", "/tmp" opts.BoltPath = fmt.Sprintf("/tmp/%d", port) opts.GithubCSEC, opts.GithubCID = "csec", "cid" + opts.GoogleCSEC, opts.GoogleCID = "csec", "cid" + opts.FacebookCSEC, opts.FacebookCID = "csec", "cid" opts.Port = port + os.Remove(opts.BoltPath + "/remark.db") + // create app app, err := New(opts) require.Nil(t, err) From 535bda89482a52aa3341b74256f1524296085264 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 22 May 2018 23:46:53 -0500 Subject: [PATCH 09/15] switch travis coverage run to CDT --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d3013444..8d6e6875 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,5 +15,7 @@ script: - docker build . after_success: + - echo 'America/Chicago' | sudo tee /etc/timezone + - sudo dpkg-reconfigure --frontend noninteractive tzdata - go test $(go list -e ./... | grep -v vendor) -covermode=count -coverprofile=coverage.out - goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN From 8291203d7a7ed0944b328bfcf851693ac1d18249 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 23 May 2018 00:30:56 -0500 Subject: [PATCH 10/15] move coveral report to docker --- .travis.yml | 17 +---------------- Dockerfile | 8 ++++++++ 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d6e6875..ce10622f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,6 @@ -language: go - -go: - - "1.10.x" - -before_install: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - install: - docker --version - docker-compose --version script: - - docker build . - -after_success: - - echo 'America/Chicago' | sudo tee /etc/timezone - - sudo dpkg-reconfigure --frontend noninteractive tzdata - - go test $(go list -e ./... | grep -v vendor) -covermode=count -coverprofile=coverage.out - - goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN + - docker build --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN . diff --git a/Dockerfile b/Dockerfile index 359e1afa..0ee24e99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ FROM umputun/baseimage:buildgo-latest as build-backend +ARG COVERALLS_TOKEN +ENV COVERALLS_TOKEN=$COVERALLS_TOKEN + WORKDIR /go/src/github.com/umputun/remark ADD app /go/src/github.com/umputun/remark/app @@ -13,6 +16,11 @@ RUN gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=ve RUN mkdir -p target && /script/coverage.sh +RUN if [ "x$COVERALLS_TOKEN" = "x" ] ; then \ + echo coverall not enabled ; \ + else go get github.com/mattn/goveralls && \ + goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi + ADD .git /go/src/github.com/umputun/remark/.git RUN go build -o remark -ldflags "-X main.revision=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) -s -w" ./app From 0d558857e3209da28e4ca950110e1b7619f13d79 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 23 May 2018 00:37:05 -0500 Subject: [PATCH 11/15] fix lack of git info for coverage --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0ee24e99..5c5536f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ WORKDIR /go/src/github.com/umputun/remark ADD app /go/src/github.com/umputun/remark/app ADD vendor /go/src/github.com/umputun/remark/vendor +ADD .git /go/src/github.com/umputun/remark/.git RUN cd app && go test $(go list -e ./... | grep -v vendor) @@ -21,7 +22,6 @@ RUN if [ "x$COVERALLS_TOKEN" = "x" ] ; then \ else go get github.com/mattn/goveralls && \ goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi -ADD .git /go/src/github.com/umputun/remark/.git RUN go build -o remark -ldflags "-X main.revision=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) -s -w" ./app From 65d0f799e7bc45b84de1dc2e52f59dbb23da1fe9 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 23 May 2018 00:46:13 -0500 Subject: [PATCH 12/15] add test for autobackup Do --- app/migrator/backup_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/app/migrator/backup_test.go b/app/migrator/backup_test.go index a8e11779..474fb051 100644 --- a/app/migrator/backup_test.go +++ b/app/migrator/backup_test.go @@ -1,6 +1,7 @@ package migrator import ( + "context" "fmt" "io" "io/ioutil" @@ -52,6 +53,26 @@ func TestMigrator_MakeBackup(t *testing.T) { assert.Equal(t, int64(52), fi.Size()) } +func TestMigrator_Do(t *testing.T) { + loc := "/tmp/remark-backups.test" + defer os.RemoveAll(loc) + os.MkdirAll(loc, 0700) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(time.Second) + cancel() + }() + + bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}, Duration: 600 * time.Millisecond} + bk.Do(ctx) + + expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102")) + fi, err := os.Lstat(expFile) + assert.NoError(t, err) + assert.Equal(t, int64(52), fi.Size()) +} + type mockExporter struct{} func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) { From 870c48a499369cbd322a623bfa71a11a5b7e64c1 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 23 May 2018 01:09:43 -0500 Subject: [PATCH 13/15] add travis arguments --- .travis.yml | 2 +- Dockerfile | 8 +++++++- app/migrator/backup_test.go | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ce10622f..17e48090 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,4 +3,4 @@ install: - docker-compose --version script: - - docker build --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN . + - docker build --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN --build-arg TRAVIS_JOB_ID=$TRAVIS_JOB_ID --build-arg TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST --build-arg TRAVIS_BRANCH=$TRAVIS_BRANCH . diff --git a/Dockerfile b/Dockerfile index 5c5536f9..b877b68a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,10 @@ FROM umputun/baseimage:buildgo-latest as build-backend ARG COVERALLS_TOKEN ENV COVERALLS_TOKEN=$COVERALLS_TOKEN +ARG TRAVIS_JOB_ID +ARG TRAVIS_PULL_REQUEST +ARG TRAVIS_BRANCH + WORKDIR /go/src/github.com/umputun/remark ADD app /go/src/github.com/umputun/remark/app @@ -22,7 +26,9 @@ RUN if [ "x$COVERALLS_TOKEN" = "x" ] ; then \ else go get github.com/mattn/goveralls && \ goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi -RUN go build -o remark -ldflags "-X main.revision=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) -s -w" ./app +RUN \ + echo "rev=$(git rev-parse --abbrev-ref HEAD)" && \ + go build -o remark -ldflags "-X main.revision=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) -s -w" ./app FROM node:9.4-alpine as build-frontend diff --git a/app/migrator/backup_test.go b/app/migrator/backup_test.go index 474fb051..55cbc231 100644 --- a/app/migrator/backup_test.go +++ b/app/migrator/backup_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMigrator_RemoveOldBackupFiles(t *testing.T) { +func TestBackup_RemoveOldBackupFiles(t *testing.T) { loc := "/tmp/remark-backups.test" defer os.RemoveAll(loc) @@ -37,7 +37,7 @@ func TestMigrator_RemoveOldBackupFiles(t *testing.T) { assert.Equal(t, "backup-site2-20171210.gz", ff[3].Name()) } -func TestMigrator_MakeBackup(t *testing.T) { +func TestBackup_MakeBackup(t *testing.T) { loc := "/tmp/remark-backups.test" defer os.RemoveAll(loc) os.MkdirAll(loc, 0700) @@ -53,7 +53,7 @@ func TestMigrator_MakeBackup(t *testing.T) { assert.Equal(t, int64(52), fi.Size()) } -func TestMigrator_Do(t *testing.T) { +func TestBackup_Do(t *testing.T) { loc := "/tmp/remark-backups.test" defer os.RemoveAll(loc) os.MkdirAll(loc, 0700) From 38fc202cca320a4b200a0cd903121aad6994618a Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 23 May 2018 01:13:47 -0500 Subject: [PATCH 14/15] clean dup in dockerfile --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b877b68a..7d63657f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,6 @@ FROM umputun/baseimage:buildgo-latest as build-backend ARG COVERALLS_TOKEN -ENV COVERALLS_TOKEN=$COVERALLS_TOKEN - ARG TRAVIS_JOB_ID ARG TRAVIS_PULL_REQUEST ARG TRAVIS_BRANCH From 6a539dabd77e5a30cd4c0666ca6eb6054695744d Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 23 May 2018 01:18:35 -0500 Subject: [PATCH 15/15] remove branch dbg --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7d63657f..41b5a4a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,7 @@ RUN if [ "x$COVERALLS_TOKEN" = "x" ] ; then \ else go get github.com/mattn/goveralls && \ goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi -RUN \ - echo "rev=$(git rev-parse --abbrev-ref HEAD)" && \ - go build -o remark -ldflags "-X main.revision=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) -s -w" ./app +RUN go build -o remark -ldflags "-X main.revision=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) -s -w" ./app FROM node:9.4-alpine as build-frontend