From ba6bab5b91688a0167d6a3696d2142c997236859 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 26 Aug 2018 21:45:04 -0500 Subject: [PATCH] Feature/cmd (#195) * support flags commands, move to cmd * fix target name * test for happy path importer * add export cmd * fix wrong import, lint warns * increase test timeout * add sellp to allow main test server to start * implement all cmds * handle backup/restore errors * fix import status check, hide secret from logs * backup cmd err tests * randimize test port * avoid dup code in Last controller * add target to make all bin archives * remove container in make * add smiple scripts to simplify commands, update readme * add docs on dockerless, enforce app user * add restore info * move last to lastCommentsScope const --- .dockerignore | 2 + .gitignore | 2 + Dockerfile | 17 +- Dockerfile.artifacts | 3 +- Makefile | 15 + README.md | 44 ++- backend/app/cmd/backup.go | 85 +++++ backend/app/cmd/backup_test.go | 71 ++++ backend/app/cmd/cmd.go | 66 ++++ backend/app/cmd/cmd_test.go | 39 ++ backend/app/cmd/import.go | 85 +++++ backend/app/cmd/import_test.go | 112 ++++++ backend/app/cmd/restore.go | 39 ++ backend/app/cmd/restore_test.go | 37 ++ backend/app/cmd/server.go | 446 ++++++++++++++++++++++ backend/app/cmd/server_test.go | 232 ++++++++++++ backend/app/cmd/testdata/import.txt | 3 + backend/app/cmd/testdata/import.txt.gz | Bin 0 -> 49 bytes backend/app/main.go | 450 +---------------------- backend/app/main_test.go | 227 ++---------- backend/app/rest/api/admin.go | 4 +- backend/app/rest/api/middleware.go | 22 ++ backend/app/rest/api/middleware_test.go | 16 + backend/app/rest/api/migrator.go | 2 +- backend/app/rest/api/rest.go | 4 +- backend/app/rest/api/rest_private.go | 4 +- backend/app/rest/api/rest_public.go | 12 +- backend/app/rest/api/rest_public_test.go | 1 + backend/app/rest/api/rest_test.go | 15 + backend/app/rest/api/rss.go | 4 +- backend/app/rest/auth/auth.go | 5 + backend/app/rest/proxy/avatar_test.go | 2 +- backend/scripts/backup.sh | 3 + backend/scripts/create-backup.sh | 9 - backend/scripts/import-disqus.sh | 5 - backend/scripts/import-wordpress.sh | 5 - backend/scripts/import.sh | 3 + backend/scripts/migrate-data.sh | 25 -- backend/scripts/restore-backup.sh | 14 - backend/scripts/restore.sh | 3 + compose-dev-backend.yml | 3 - compose-dev-frontend.yml | 3 - docker-compose.yml | 3 - start.sh => entrypoint.sh | 14 +- 44 files changed, 1400 insertions(+), 756 deletions(-) create mode 100644 backend/app/cmd/backup.go create mode 100644 backend/app/cmd/backup_test.go create mode 100644 backend/app/cmd/cmd.go create mode 100644 backend/app/cmd/cmd_test.go create mode 100644 backend/app/cmd/import.go create mode 100644 backend/app/cmd/import_test.go create mode 100644 backend/app/cmd/restore.go create mode 100644 backend/app/cmd/restore_test.go create mode 100644 backend/app/cmd/server.go create mode 100644 backend/app/cmd/server_test.go create mode 100644 backend/app/cmd/testdata/import.txt create mode 100644 backend/app/cmd/testdata/import.txt.gz create mode 100644 backend/scripts/backup.sh delete mode 100755 backend/scripts/create-backup.sh delete mode 100644 backend/scripts/import-disqus.sh delete mode 100755 backend/scripts/import-wordpress.sh create mode 100644 backend/scripts/import.sh delete mode 100755 backend/scripts/migrate-data.sh delete mode 100755 backend/scripts/restore-backup.sh create mode 100644 backend/scripts/restore.sh rename start.sh => entrypoint.sh (63%) diff --git a/.dockerignore b/.dockerignore index 3cd880c4..e6f1136f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,3 +19,5 @@ debug debug.test *.prof *.test +/bin/ +remark42 diff --git a/.gitignore b/.gitignore index e1385214..1ab85900 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ debug.test /rest-client.env.json .DS_Store .mongo +remark42 +/bin/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2f4a1a24..5006297e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ RUN \ echo "runs outside of drone" && version="local"; \ else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \ echo "version=$version" && \ - go build -o remark -ldflags "-X main.revision=${version} -s -w" ./app + go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app FROM node:10.6-alpine as build-frontend-deps @@ -88,16 +88,19 @@ FROM umputun/baseimage:app-latest WORKDIR /srv -ADD backend/scripts/*.sh /srv/ -ADD start.sh /srv/start.sh -RUN chmod +x /srv/*.sh +ADD entrypoint.sh /entrypoint.sh +ADD backend/scripts/backup.sh /usr/local/bin/backup +ADD backend/scripts/restore.sh /usr/local/bin/restore +ADD backend/scripts/import.sh /usr/local/bin/import +RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import -COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark /srv/ +COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42 COPY --from=build-frontend /srv/web/public/ /srv/web RUN chown -R app:app /srv +RUN ln -s /srv/remark42 /usr/bin/remark42 EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1 -CMD ["/srv/start.sh"] -ENTRYPOINT ["/init.sh"] +CMD ["server"] +ENTRYPOINT ["/entrypoint.sh"] diff --git a/Dockerfile.artifacts b/Dockerfile.artifacts index 2ab65127..1b739f87 100644 --- a/Dockerfile.artifacts +++ b/Dockerfile.artifacts @@ -59,7 +59,7 @@ RUN \ tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \ tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \ tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \ - zip remark${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md + zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md # upload to github RUN \ @@ -84,4 +84,5 @@ RUN \ FROM alpine COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/ +RUN ls -la /artifacts/* CMD ["sleep", "100"] diff --git a/Makefile b/Makefile index fc5b9690..27a983cf 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,24 @@ ARCH=amd64 bin: docker build -f Dockerfile.artifacts -t remark42.bin . + - @docker rm -f remark42.bin 2>/dev/null || exit 0 docker run -d --name=remark42.bin remark42.bin docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42 docker rm -f remark42.bin docker: docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true . + +deploy: + docker build -f Dockerfile.artifacts -t remark42.bin . + - @docker rm -f remark42.bin 2>/dev/null || exit 0 + - @mkdir -p bin + docker run -d --name=remark42.bin remark42.bin + docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz + docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz + docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz + docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz + docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip + docker rm -f remark42.bin + + diff --git a/README.md b/README.md index 5ba62cc9..b8756152 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi - [Install](#install) - [Backend](#backend) + - [With Docker](#with-docker) + - [Without docker](#without-docker) - [Parameters](#parameters) - [Required parameters](#required-parameters) - [Register oauth2 providers](#register-oauth2-providers) @@ -35,8 +37,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi - [Initial import from WordPress](#initial-import-from-wordpress) - [Backup and restore](#backup-and-restore) - [Automatic backups](#automatic-backups) - - [Schema migration](#schema-migration) - [Manual backup](#manual-backup) + - [Restore from backup](#restore-from-backup) - [Backup format](#backup-format) - [Admin users](#admin-users) - [Setup on your website](#setup-on-your-website) @@ -62,11 +64,21 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi ### Backend +#### With Docker + +_this is the recommended way to run remark42_ + * copy provided `docker-compose.yml` and customize for your needs -* prepare user id for container `` export USER=`id -u $USER` `` * make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments -* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d` -* alternatively compile from sources - `docker-compose build && docker-compose up -d` +* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d` +* alternatively compile from the sources - `docker-compose build && docker-compose up -d` + +#### Without docker + +* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads) +* unpack with `gunzip` (linux, mac os) or with `zip` (windows) +* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080` +* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]` #### Parameters @@ -139,7 +151,6 @@ services: - SECRET=abcd-123456-xyz-$%^& # secret key - AUTH_GITHUB_CID=12345667890 # oauth2 client ID - AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret - - USER=1001 # UID on the host machine, i.e `id -u` volumes: - ./var:/srv/var # persistent volume to store all remark42 data ``` @@ -200,13 +211,13 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co 1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details. 2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip .xml.gz`. -3. Run import command - `docker-compose exec remark42 /srv/import-disqus.sh .xml ` +3. Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}` #### Initial import from WordPress 1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. 2. Move this file to your remark42 host within `./var` -3. Run import command - `docker-compose exec remark42 /srv/import-wordpress.sh .xml ` +3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}` #### Backup and restore @@ -215,20 +226,19 @@ Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./ For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container: -`docker-compose exec remark42 /srv/restore-backup.sh {backup-filename.gz} {your site id}` - -##### Schema migration - -One special case for backup/restore is schema migration. Some versions or remark42 may extend or change the schema -and for such upgrades migration required. Provided migration script `scripts/migrate-data.sh` makes a fresh backup and then loads it back to your remark42 instance. - -`docker-compose exec remark42 /srv/migrate-data.sh {your site id}` +`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}` ##### Manual backup -In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` +In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default. -`docker-compose exec remark42 /srv/create-backup.sh {your site id}` +`docker exec -it remark42 backup -s {your site id}` + +##### Restore from backup + +Restore will clean all comments first and then will processed with complete import from a given file. + +`docker exec -it remark42 restore -f {backup file name} -s {your site id}` ##### Backup format diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go new file mode 100644 index 00000000..815e7794 --- /dev/null +++ b/backend/app/cmd/backup.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "time" + + "github.com/pkg/errors" +) + +// BackupCommand set of flags and command for export +// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. /) BACKUP_PATH ignored. +type BackupCommand struct { + ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` + ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"` + Site string `long:"site" env:"SITE" default:"remark" description:"site name"` + SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` +} + +// Execute runs export with ExportCommand parameters, entry point for "export" command +func (ec *BackupCommand) Execute(args []string) error { + log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site) + resetEnv("SECRET") + + fp := fileParser{site: ec.Site, path: ec.ExportPath, file: ec.ExportFile} + fname, err := fp.parse(time.Now()) + if err != nil { + return err + } + + log.Printf("[DEBUG] export file %s", fname) + + // prepare http client + client := http.Client{} + exportURL := fmt.Sprintf("%s/api/v1/admin/export?site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret) + req, err := http.NewRequest(http.MethodGet, exportURL, nil) + if err != nil { + return errors.Wrapf(err, "can't make export request for %s", exportURL) + } + ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout) + defer cancel() + + // get with timeout + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrapf(err, "request failed for %s", exportURL) + } + defer func() { + if err = resp.Body.Close(); err != nil { + log.Printf("[WARN] failed to close response, %s", err) + } + }() + + if resp.StatusCode >= 300 { + body, e := ioutil.ReadAll(resp.Body) + if e != nil { + body = []byte("") + } + return errors.Errorf("error response %q, %s", resp.Status, body) + } + + fh, err := os.Create(fname) + if err != nil { + return errors.Wrapf(err, "can't create backup file %s", fname) + } + defer func() { + if err = fh.Close(); err != nil { + log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err) + } + }() + + if _, err = io.Copy(fh, resp.Body); err != nil { + return errors.Wrapf(err, "failed to write backup file %s", fname) + } + + log.Printf("[INFO] export completed, file %s", fname) + return nil +} diff --git a/backend/app/cmd/backup_test.go b/backend/app/cmd/backup_test.go new file mode 100644 index 00000000..e3d56c11 --- /dev/null +++ b/backend/app/cmd/backup_test.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBackup_Execute(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/export") + assert.Equal(t, "GET", r.Method) + fmt.Fprint(w, "blah\nblah2\n12345678\n") + })) + defer ts.Close() + + cmd := BackupCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", + "--file={{.SITE}}-test.export", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) + defer os.Remove("/tmp/remark-test.export") + + data, err := ioutil.ReadFile("/tmp/remark-test.export") + require.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(data)) +} + +func TestBackup_ExecuteFailedStatus(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/export") + assert.Equal(t, "GET", r.Method) + w.WriteHeader(400) + fmt.Fprint(w, "some error") + })) + defer ts.Close() + + cmd := BackupCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", + "--file={{.SITE}}-test.export", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.EqualError(t, err, `error response "400 Bad Request", some error`) +} + +func TestBackup_ExecuteFailedWrite(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/export") + assert.Equal(t, "GET", r.Method) + fmt.Fprint(w, "blah\nblah2\n12345678\n") + })) + defer ts.Close() + + cmd := BackupCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", + "--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`) +} diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go new file mode 100644 index 00000000..15bee7c1 --- /dev/null +++ b/backend/app/cmd/cmd.go @@ -0,0 +1,66 @@ +// Package cmd has all top-level commands dispatched by main's flag.Parse +// The entry point of each command is Execute function +package cmd + +import ( + "bytes" + "log" + "os" + "path/filepath" + "strings" + "text/template" + "time" + + "github.com/pkg/errors" +) + +// fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format +type fileParser struct { + site string + file string + path string +} + +// parse apply template and also concat path and file. In case if file contains path separator path will be ignored +func (p *fileParser) parse(now time.Time) (string, error) { + + fileTemplate := struct { + YYYYMMDD string + YYYY string + YYYYMM string + MM string + DD string + TS string + UNIX int64 + SITE string + }{ + YYYYMMDD: now.Format("20060102"), + YYYY: now.Format("2006"), + YYYYMM: now.Format("200601"), + MM: now.Format("01"), + DD: now.Format("02"), + UNIX: now.Unix(), + SITE: p.site, + TS: now.Format("20060102T150405"), + } + + bb := bytes.Buffer{} + fname := p.file + if !strings.Contains(p.file, string(filepath.Separator)) { + fname = filepath.Join(p.path, p.file) + } + + if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil { + return "", errors.Wrapf(err, "failed to parse %q", fname) + } + return bb.String(), nil +} + +// resetEnv clears sensitive env vars +func resetEnv(envs ...string) { + for _, env := range envs { + if err := os.Unsetenv(env); err != nil { + log.Printf("[WARN] can't unset env %s, %s", env, err) + } + } +} diff --git a/backend/app/cmd/cmd_test.go b/backend/app/cmd/cmd_test.go new file mode 100644 index 00000000..de20b50d --- /dev/null +++ b/backend/app/cmd/cmd_test.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestExport_ParseFileName(t *testing.T) { + tbl := []struct { + p fileParser + res string + err bool + }{ + {fileParser{}, "", false}, + {fileParser{path: "/tmp/blah", file: "fname.gz"}, "/tmp/blah/fname.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYYMMDD}}.gz"}, + "/tmp/blah/fname-remark-20180821.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"}, + "/tmp/blah/fname-remark-2018-08.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"}, + "/tmp/fname-remark-2018-08.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.TS}}.gz"}, + "/tmp/fname-remark-20180821T212615.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.XXX}}-{{.YYYY}}-{{.MM}}.gz"}, + "", true}, + } + + now := time.Date(2018, 8, 21, 21, 26, 15, 0, time.UTC) + for i, tt := range tbl { + r, err := tt.p.parse(now) + if tt.err { + assert.NotNil(t, err) + continue + } + assert.Equal(t, tt.res, r, "check #%d", i) + } +} diff --git a/backend/app/cmd/import.go b/backend/app/cmd/import.go new file mode 100644 index 00000000..13b4ef8b --- /dev/null +++ b/backend/app/cmd/import.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strings" + "time" + + "github.com/pkg/errors" +) + +// ImportCommand set of flags and command for import +type ImportCommand struct { + InputFile string `short:"f" long:"file" description:"input file name" required:"true"` + Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` + Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` + SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` + + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` +} + +// Execute runs import with ImportCommand parameters, entry point for "import" command +func (ic *ImportCommand) Execute(args []string) error { + log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site) + resetEnv("SECRET") + + reader, err := ic.reader(ic.InputFile) + if err != nil { + return errors.Wrapf(err, "can't open import file %s", ic.InputFile) + } + + client := http.Client{} + importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s", ic.URL, ic.Site, ic.Provider, ic.SharedSecret) + req, err := http.NewRequest(http.MethodPost, importURL, reader) + if err != nil { + return errors.Wrapf(err, "can't make import request for %s", importURL) + } + + ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout) + defer cancel() + + resp, err := client.Do(req.WithContext(ctx)) // closes reader + if err != nil { + return errors.Wrapf(err, "request failed for %s", importURL) + } + defer func() { + if err = resp.Body.Close(); err != nil { + log.Printf("[WARN] failed to close response, %s", err) + } + }() + if resp.StatusCode >= 300 { + return errors.Errorf("error response %s (%d)", resp.Status, resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "can't get response from importer") + } + + log.Printf("[INFO] import completed, status=%d, %s", resp.StatusCode, string(body)) + return nil +} + +// reader returns reader for file. For .gz file wraps with gunzip +func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) { + inpFile, err := os.Open(inp) + if err != nil { + return nil, errors.Wrapf(err, "import failed, can't open %s", inp) + } + + reader = inpFile + if strings.HasSuffix(ic.InputFile, ".gz") { + if reader, err = gzip.NewReader(inpFile); err != nil { + return nil, errors.Wrap(err, "can't make gz reader") + } + } + return reader, nil +} diff --git a/backend/app/cmd/import_test.go b/backend/app/cmd/import_test.go new file mode 100644 index 00000000..30188da5 --- /dev/null +++ b/backend/app/cmd/import_test.go @@ -0,0 +1,112 @@ +package cmd + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImport_Execute(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + body, err := ioutil.ReadAll(r.Body) + assert.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(body)) + + fmt.Fprintln(w, "some response") + fmt.Fprintln(w, string(body)) + })) + defer ts.Close() + + cmd := ImportCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) + + cmd = ImportCommand{} + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt.gz", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) +} + +func TestImport_ExecuteFailed(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + fmt.Fprintln(w, "some response") + })) + defer ts.Close() + + cmd := ImportCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import-no.txt", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + t.Log(err) + assert.NotNil(t, err, "fail on no such file") + assert.True(t, strings.Contains(err.Error(), "no such file or directory")) + + cmd = ImportCommand{} + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", + "--url=http://127.0.0.1:12345"}) + require.Nil(t, err) + err = cmd.Execute(nil) + t.Log(err) + assert.NotNil(t, err, "fail on connection refused") + assert.True(t, strings.Contains(err.Error(), "connection refused")) + + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%+v", r) + w.WriteHeader(400) + fmt.Fprintln(w, "some response with 400") + })) + defer ts2.Close() + cmd = ImportCommand{} + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts2.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + t.Log(err) + assert.NotNil(t, err) +} + +func TestImport_ExecuteTimeout(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + body, err := ioutil.ReadAll(r.Body) + assert.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(body)) + time.Sleep(500 * time.Millisecond) + fmt.Fprintln(w, "some response") + fmt.Fprintln(w, string(body)) + + })) + defer ts.Close() + + cmd := ImportCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", + "--url=" + ts.URL, "--timeout=300ms"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "deadline exceeded")) +} diff --git a/backend/app/cmd/restore.go b/backend/app/cmd/restore.go new file mode 100644 index 00000000..c8e42461 --- /dev/null +++ b/backend/app/cmd/restore.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "log" + "time" +) + +// RestoreCommand set of flags and command for restore from backup +type RestoreCommand struct { + ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` + ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"` + + Site string `long:"site" env:"SITE" default:"remark" description:"site name"` + SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` +} + +// Execute runs import with RestoreCommand parameters, entry point for "restore" command +// uses ImportCommand with constructed full file name +func (rc *RestoreCommand) Execute(args []string) error { + log.Printf("[INFO] restore %s, site %s", rc.ImportFile, rc.Site) + resetEnv("SECRET") + + fp := fileParser{site: rc.Site, path: rc.ImportPath, file: rc.ImportFile} + fname, err := fp.parse(time.Now()) + if err != nil { + return err + } + importer := ImportCommand{ + InputFile: fname, + Site: rc.Site, + Provider: "native", + SharedSecret: rc.SharedSecret, + Timeout: rc.Timeout, + URL: rc.URL, + } + return importer.Execute(args) +} diff --git a/backend/app/cmd/restore_test.go b/backend/app/cmd/restore_test.go new file mode 100644 index 00000000..820968fb --- /dev/null +++ b/backend/app/cmd/restore_test.go @@ -0,0 +1,37 @@ +package cmd + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRestore_Execute(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + assert.Equal(t, "native", r.URL.Query().Get("provider")) + body, err := ioutil.ReadAll(r.Body) + assert.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(body)) + + fmt.Fprintln(w, "some response") + fmt.Fprintln(w, string(body)) + })) + defer ts.Close() + + cmd := RestoreCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=testdata", "--file=import.txt", + "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) +} diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go new file mode 100644 index 00000000..c85c3889 --- /dev/null +++ b/backend/app/cmd/server.go @@ -0,0 +1,446 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/coreos/bbolt" + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/migrator" + "github.com/umputun/remark/backend/app/rest/api" + "github.com/umputun/remark/backend/app/rest/auth" + "github.com/umputun/remark/backend/app/rest/cache" + "github.com/umputun/remark/backend/app/rest/proxy" + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/avatar" + "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" + "github.com/umputun/remark/backend/app/store/service" +) + +// ServerOpts with command line flags and env +type ServerOpts struct { + RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` + SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` + + Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` + Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` + Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` + Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` + Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` + + Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` + Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` + AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"` + DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"` + BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"` + MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` + ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"` + MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"` + LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"` + CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"` + ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"` + EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"` + Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` + WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` + // Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` + + Auth struct { + TTL struct { + JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"` + Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"` + } `group:"ttl" namespace:"ttl" env-namespace:"TTL"` + Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"` + Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"` + Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"` + Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"` + Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"` + } `group:"auth" namespace:"auth" env-namespace:"AUTH"` +} + +// AuthGroup defines options group for auth params +type AuthGroup struct { + CID string `long:"cid" env:"CID" description:"OAuth client ID"` + CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"` +} + +// StoreGroup defines options group for store params +type StoreGroup struct { + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"` + Bolt struct { + Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` + Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` + } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` +} + +// AvatarGroup defines options group for avatar params +type AvatarGroup struct { + Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"` + FS struct { + Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` + } `group:"fs" namespace:"fs" env-namespace:"FS"` + RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` +} + +// CacheGroup defines options group for cache params +type CacheGroup struct { + Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"` + Max struct { + Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` + Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` + Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"` + } `group:"max" namespace:"max" env-namespace:"MAX"` +} + +// MongoGroup holds all mongo params, used by store, avatar and cache +type MongoGroup struct { + URL string `long:"url" env:"URL" description:"mongo url"` + DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` +} + +// KeyGroup defines options group for key params +type KeyGroup struct { + Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` +} + +// Revision sets from main +var Revision = "unknown" + +// serverApp holds all active objects +type serverApp struct { + *ServerOpts + restSrv *api.Rest + migratorSrv *api.Migrator + exporter migrator.Exporter + devAuth *auth.DevAuthServer + dataService *service.DataStore + terminated chan struct{} +} + +// Execute is the entry point for "server" command, called by flag parser +func (s *ServerOpts) Execute(args []string) error { + log.Print("[INFO] start remark42 server") + resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC") + + ctx, cancel := context.WithCancel(context.Background()) + go func() { // catch signal and invoke graceful termination + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop + log.Print("[WARN] interrupt signal") + cancel() + }() + + app, err := newServerApp(s) + if err != nil { + log.Fatalf("[ERROR] failed to setup application, %+v", err) + } + if err = app.run(ctx); err != nil { + log.Printf("[INFO] remark terminated with error %+v", err) + return err + } + log.Printf("[INFO] remark terminated") + return nil +} + +// newServerApp prepares application and return it with all active parts +// doesn't start anything +func newServerApp(opts *ServerOpts) (*serverApp, error) { + + if err := makeDirs(opts.BackupLocation); err != nil { + return nil, err + } + + if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") { + return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL) + } + + storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites) + if err != nil { + return nil, err + } + + keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret) + if err != nil { + return nil, err + } + + dataService := &service.DataStore{ + Interface: storeEngine, + EditDuration: opts.EditDuration, + KeyStore: keyStore, + MaxCommentSize: opts.MaxCommentSize, + Admins: opts.Admins, + } + + loadingCache, err := makeCache(opts.Cache, opts.Mongo) + if err != nil { + return nil, err + } + + // token TTL is 5 minutes, inactivity interval 7+ days by default + jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) + + avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) + if err != nil { + return nil, errors.Wrap(err, "failed to make avatar store") + } + avatarProxy := &proxy.Avatar{ + Store: avatarStore, + RoutePath: "/api/v1/avatar", + RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), + } + + exporter := &migrator.Remark{DataStore: dataService} + + migr := &api.Migrator{ + Version: Revision, + Cache: loadingCache, + NativeImporter: &migrator.Remark{DataStore: dataService}, + DisqusImporter: &migrator.Disqus{DataStore: dataService}, + WordPressImporter: &migrator.WordPress{DataStore: dataService}, + NativeExported: &migrator.Remark{DataStore: dataService}, + KeyStore: keyStore, + } + + authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) + imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} + commentFormatter := store.NewCommentFormatter(imgProxy) + + srv := &api.Rest{ + Version: Revision, + DataService: dataService, + Exporter: exporter, + WebRoot: opts.WebRoot, + RemarkURL: opts.RemarkURL, + ImageProxy: imgProxy, + CommentFormatter: commentFormatter, + AvatarProxy: avatarProxy, + ReadOnlyAge: opts.ReadOnlyAge, + SharedSecret: opts.SharedSecret, + Authenticator: auth.Authenticator{ + JWTService: jwtService, + AdminEmail: opts.AdminEmail, + Providers: authProviders, + DevPasswd: opts.DevPasswd, + PermissionChecker: dataService, + }, + Cache: loadingCache, + } + + // no admin email, use admin@domain + if srv.Authenticator.AdminEmail == "" { + if u, err := url.Parse(opts.RemarkURL); err == nil { + srv.Authenticator.AdminEmail = "admin@" + u.Host + } + } + + srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore + + var devAuth *auth.DevAuthServer + if opts.Auth.Dev { + devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]} + } + + tch := make(chan struct{}) + return &serverApp{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService, + ServerOpts: opts, terminated: tch}, nil +} + +// Run all application objects +func (a *serverApp) run(ctx context.Context) error { + if a.DevPasswd != "" { + log.Printf("[WARN] running in dev mode") + } + + go func() { + // shutdown on context cancellation + <-ctx.Done() + a.restSrv.Shutdown() + a.migratorSrv.Shutdown() + if a.devAuth != nil { + a.devAuth.Shutdown() + } + if e := a.dataService.Close(); e != nil { + log.Printf("[WARN] failed to close store, %s", e) + } + + }() + a.activateBackup(ctx) // runs in goroutine for each site + go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only + if a.Auth.Dev { + go a.devAuth.Run() // dev oauth2 server on :8084 + } + a.restSrv.Run(a.Port) + close(a.terminated) + return nil +} + +// Wait for application completion (termination) +func (a *serverApp) Wait() { + <-a.terminated +} + +// activateBackup runs background backups for each site +func (a *serverApp) activateBackup(ctx context.Context) { + for _, siteID := range a.Sites { + backup := migrator.AutoBackup{ + Exporter: a.exporter, + BackupLocation: a.BackupLocation, + SiteID: siteID, + KeepMax: a.MaxBackupFiles, + Duration: 24 * time.Hour, + } + go backup.Do(ctx) + } +} + +// makeDataStore creates store for all sites +func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) { + switch group.Type { + case "bolt": + if err = makeDirs(group.Bolt.Path); err != nil { + return nil, errors.Wrap(err, "failed to create bolt store") + } + sites := []engine.BoltSite{} + for _, site := range siteNames { + sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)}) + } + result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...) + case "mongo": + mgServer, e := makeMongo(mg) + if e != nil { + return result, errors.Wrap(e, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, mg.DB, "") + result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) + default: + return nil, errors.Errorf("unsupported store type %s", group.Type) + } + return result, errors.Wrap(err, "can't initialize data store") +} + +func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { + switch group.Type { + case "fs": + if err := makeDirs(group.FS.Path); err != nil { + return nil, err + } + return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil + case "mongo": + mgServer, err := makeMongo(mg) + if err != nil { + return nil, errors.Wrap(err, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, mg.DB, "") + return avatar.NewGridFS(conn, group.RszLmt), nil + } + return nil, errors.Errorf("unsupported avatar store type %s", group.Type) +} + +func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) { + switch group.Type { + case "shared": + return keys.NewStaticStore(sharedSecret), nil + default: + return nil, errors.Errorf("unsupported key store type %s", group.Type) + } +} + +func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { + switch group.Type { + case "mem": + return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), + cache.MaxKeys(group.Max.Items)) + case "mongo": + mgServer, err := makeMongo(mg) + if err != nil { + return nil, errors.Wrap(err, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, mg.DB, "cache") + return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), + cache.MaxKeys(group.Max.Items)) + } + return nil, errors.Errorf("unsupported cache type %s", group.Type) +} + +// mkdir -p for all dirs +func makeDirs(dirs ...string) error { + + // exists returns whether the given file or directory exists or not + exists := func(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return true, err + } + + for _, dir := range dirs { + ex, err := exists(dir) + if err != nil { + return errors.Wrapf(err, "can't check directory status for %s", dir) + } + if !ex { + if e := os.MkdirAll(dir, 0700); e != nil { + return errors.Wrapf(err, "can't make directory %s", dir) + } + } + } + return nil +} + +func makeMongo(mg MongoGroup) (result *mongo.Server, err error) { + if mg.URL == "" { + return nil, errors.New("no mongo URL provided") + } + return mongo.NewServerWithURL(mg.URL, 10*time.Second) +} + +func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts *ServerOpts) []auth.Provider { + + makeParams := func(cid, secret string) auth.Params { + return auth.Params{ + JwtService: jwtService, + AvatarProxy: avatarProxy, + RemarkURL: opts.RemarkURL, + Cid: cid, + Csecret: secret, + PermissionChecker: ds, + } + } + + providers := []auth.Provider{} + if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" { + providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC))) + } + if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" { + providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC))) + } + if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" { + providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC))) + } + if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" { + providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC))) + } + if opts.Auth.Dev { + providers = append(providers, auth.NewDev(makeParams("", ""))) + } + + if len(providers) == 0 { + log.Printf("[WARN] no auth providers defined") + } + return providers +} diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go new file mode 100644 index 00000000..5f25501e --- /dev/null +++ b/backend/app/cmd/server_test.go @@ -0,0 +1,232 @@ +package cmd + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strings" + "syscall" + "testing" + "time" + + "github.com/globalsign/mgo" + "github.com/go-pkgz/mongo" + "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServerApp(t *testing.T) { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + o.Port = 18080 + return o + }) + + go func() { _ = app.run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:18080/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + // add comment + resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json", + strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`)) + require.Nil(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + body, _ = ioutil.ReadAll(resp.Body) + t.Log(string(body)) + + assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email") + + app.Wait() +} + +func TestServerApp_DevMode(t *testing.T) { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + o.Port = 18085 + o.DevPasswd = "password" + o.Auth.Dev = true + return o + }) + + go func() { _ = app.run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider") + assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider") + // send ping + resp, err := http.Get("http://localhost:18085/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + app.Wait() +} + +func TestServerApp_WithMongo(t *testing.T) { + + mongoURL := os.Getenv("MONGO_TEST") + if mongoURL == "" { + mongoURL = "mongodb://localhost:27017/test" + } + if mongoURL == "skip" { + t.Skip("skip mongo app test") + } + + opts := ServerOpts{} + // prepare options + p := flags.NewParser(&opts, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com", + "--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) + require.Nil(t, err) + opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" + opts.BackupLocation = "/tmp" + + // create app + app, err := newServerApp(&opts) + require.Nil(t, err) + + defer func() { + s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second) + assert.NoError(t, err) + conn := mongo.NewConnection(s, "test_remark", "") + _ = conn.WithDB(func(dbase *mgo.Database) error { + assert.NoError(t, dbase.DropDatabase()) + return nil + }) + }() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Second) + log.Print("[TEST] terminate app") + cancel() + }() + go func() { _ = app.run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:12345/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + app.Wait() +} + +func TestServerApp_Failed(t *testing.T) { + opts := ServerOpts{} + p := flags.NewParser(&opts, flags.Default) + + // RO bolt location + _, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", + "--store.bolt.path=/dev/null"}) + assert.Nil(t, err) + _, err = newServerApp(&opts) + assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ + "open /dev/null/remark.db: not a directory") + t.Log(err) + + // RO backup location + opts = ServerOpts{} + _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp", + "--backup=/dev/null/not-writable"}) + assert.Nil(t, err) + _, err = newServerApp(&opts) + assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory") + t.Log(err) + + // invalid url + opts = ServerOpts{} + _, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"}) + assert.Nil(t, err) + _, err = newServerApp(&opts) + assert.EqualError(t, err, "invalid remark42 url demo.remark42.com") + t.Log(err) + + opts = ServerOpts{} + _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"}) + assert.NotNil(t, err, "blah is invalid type") + + opts.Store.Type = "blah" + _, err = newServerApp(&opts) + assert.EqualError(t, err, "unsupported store type blah") + t.Log(err) +} + +func TestServerApp_Shutdown(t *testing.T) { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + o.Port = 18090 + return o + }) + st := time.Now() + err := app.run(ctx) + assert.Nil(t, err) + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") + app.Wait() +} + +func TestServerApp_MainSignal(t *testing.T) { + + go func() { + time.Sleep(100 * time.Millisecond) + err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM) + require.Nil(t, err) + }() + st := time.Now() + + s := ServerOpts{} + p := flags.NewParser(&s, flags.Default) + args := []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", + "--port=18100", "--url=https://demo.remark42.com"} + _, err := p.ParseArgs(args) + require.Nil(t, err) + s.Execute(args) + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") +} + +func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerOpts) ServerOpts) (*serverApp, context.Context) { + opts := ServerOpts{} + // prepare options + p := flags.NewParser(&opts, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"}) + require.Nil(t, err) + opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp" + opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port) + opts.Store.Bolt.Timeout = 10 * time.Second + opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" + opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid" + opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid" + opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid" + opts.BackupLocation = "/tmp" + opts = fn(opts) + + os.Remove(opts.Store.Bolt.Path + "/remark.db") + + // create app + app, err := newServerApp(&opts) + require.Nil(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(duration) + log.Print("[TEST] terminate app") + cancel() + }() + return app, ctx +} diff --git a/backend/app/cmd/testdata/import.txt b/backend/app/cmd/testdata/import.txt new file mode 100644 index 00000000..f9920896 --- /dev/null +++ b/backend/app/cmd/testdata/import.txt @@ -0,0 +1,3 @@ +blah +blah2 +12345678 diff --git a/backend/app/cmd/testdata/import.txt.gz b/backend/app/cmd/testdata/import.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..8691c9afcc4664add0b040afccf1bb151c1ab3d7 GIT binary patch literal 49 zcmb2|=HNJ$RTIs?oS9pYUsR%3Qc=R-eaiF911}~ax0p7!V`|INo{7aU%$}z1Bf`MI F006Z35YPYs literal 0 HcmV?d00001 diff --git a/backend/app/main.go b/backend/app/main.go index 699a770b..df619011 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -1,460 +1,48 @@ package main import ( - "context" "fmt" "log" - "net/url" "os" - "os/signal" - "strings" - "syscall" - "time" - "github.com/coreos/bbolt" - "github.com/go-pkgz/mongo" "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" - "github.com/pkg/errors" - "github.com/umputun/remark/backend/app/migrator" - "github.com/umputun/remark/backend/app/rest/api" - "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" - "github.com/umputun/remark/backend/app/rest/proxy" - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/avatar" - "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" - "github.com/umputun/remark/backend/app/store/service" + "github.com/umputun/remark/backend/app/cmd" ) -// Opts with command line flags and env +// Opts has all commands type Opts struct { - RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` - SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` + ServerCmd cmd.ServerOpts `command:"server"` + ImportCmd cmd.ImportCommand `command:"import"` + BackupCmd cmd.BackupCommand `command:"backup"` + RestoreCmd cmd.RestoreCommand `command:"restore"` - Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` - Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` - Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` - Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` - Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` - - Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` - Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` - AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"` - DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"` - BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"` - MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` - ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"` - MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"` - LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"` - CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"` - ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"` - EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"` - Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` - WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` - Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` - - Auth struct { - TTL struct { - JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"` - Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"` - } `group:"ttl" namespace:"ttl" env-namespace:"TTL"` - Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"` - Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"` - Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"` - Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"` - Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"` - } `group:"auth" namespace:"auth" env-namespace:"AUTH"` -} - -// AuthGroup defines options group for auth params -type AuthGroup struct { - CID string `long:"cid" env:"CID" description:"OAuth client ID"` - CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"` -} - -// StoreGroup defines options group for store params -type StoreGroup struct { - Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"` - Bolt struct { - Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` - Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` - } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` -} - -// AvatarGroup defines options group for avatar params -type AvatarGroup struct { - Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"` - FS struct { - Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` - } `group:"fs" namespace:"fs" env-namespace:"FS"` - RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` -} - -// CacheGroup defines options group for cache params -type CacheGroup struct { - Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"` - Max struct { - Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` - Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` - Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"` - } `group:"max" namespace:"max" env-namespace:"MAX"` -} - -// MongoGroup holds all mongo params, used by store, avatar and cache -type MongoGroup struct { - URL string `long:"url" env:"URL" description:"mongo url"` - DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` -} - -// KeyGroup defines options group for key params -type KeyGroup struct { - Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` + Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` } var revision = "unknown" -// Application holds all active objects -type Application struct { - Opts - restSrv *api.Rest - migratorSrv *api.Migrator - exporter migrator.Exporter - devAuth *auth.DevAuthServer - dataService *service.DataStore - terminated chan struct{} -} - func main() { - fmt.Printf("remark %s\n", revision) + fmt.Printf("remark42 %s\n", revision) + cmd.Revision = revision var opts Opts p := flags.NewParser(&opts, flags.Default) - if _, e := p.ParseArgs(os.Args[1:]); e != nil { - os.Exit(1) - } - - setupLog(opts.Dbg) - log.Print("[INFO] started remark") - resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC") - - ctx, cancel := context.WithCancel(context.Background()) - go func() { // catch signal and invoke graceful termination - stop := make(chan os.Signal, 1) - signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - <-stop - log.Print("[WARN] interrupt signal") - cancel() - }() - - app, err := New(opts) - if err != nil { - log.Fatalf("[ERROR] failed to setup application, %+v", err) - } - err = app.Run(ctx) - log.Printf("[INFO] remark terminated %s", err) -} - -// New prepares application and return it with all active parts -// doesn't start anything -func New(opts Opts) (*Application, error) { - - if err := makeDirs(opts.BackupLocation); err != nil { - return nil, err - } - - if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") { - return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL) - } - - storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites) - if err != nil { - return nil, err - } - - keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret) - if err != nil { - return nil, err - } - - dataService := &service.DataStore{ - Interface: storeEngine, - EditDuration: opts.EditDuration, - KeyStore: keyStore, - MaxCommentSize: opts.MaxCommentSize, - Admins: opts.Admins, - } - - loadingCache, err := makeCache(opts.Cache, opts.Mongo) - if err != nil { - return nil, err - } - - // token TTL is 5 minutes, inactivity interval 7+ days by default - jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) - - avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) - if err != nil { - return nil, errors.Wrap(err, "failed to make avatar store") - } - avatarProxy := &proxy.Avatar{ - Store: avatarStore, - RoutePath: "/api/v1/avatar", - RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), - } - - exporter := &migrator.Remark{DataStore: dataService} - - migr := &api.Migrator{ - Version: revision, - Cache: loadingCache, - NativeImporter: &migrator.Remark{DataStore: dataService}, - DisqusImporter: &migrator.Disqus{DataStore: dataService}, - WordPressImporter: &migrator.WordPress{DataStore: dataService}, - NativeExported: &migrator.Remark{DataStore: dataService}, - KeyStore: keyStore, - } - - authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) - imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} - commentFormatter := store.NewCommentFormatter(imgProxy) - - srv := &api.Rest{ - Version: revision, - DataService: dataService, - Exporter: exporter, - WebRoot: opts.WebRoot, - RemarkURL: opts.RemarkURL, - ImageProxy: imgProxy, - CommentFormatter: commentFormatter, - AvatarProxy: avatarProxy, - ReadOnlyAge: opts.ReadOnlyAge, - SharedSecret: opts.SharedSecret, - Authenticator: auth.Authenticator{ - JWTService: jwtService, - AdminEmail: opts.AdminEmail, - Providers: authProviders, - DevPasswd: opts.DevPasswd, - PermissionChecker: dataService, - }, - Cache: loadingCache, - } - - // no admin email, use admin@domain - if srv.Authenticator.AdminEmail == "" { - if u, err := url.Parse(opts.RemarkURL); err == nil { - srv.Authenticator.AdminEmail = "admin@" + u.Host - } - } - - srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore - - var devAuth *auth.DevAuthServer - if opts.Auth.Dev { - devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]} - } - - tch := make(chan struct{}) - return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService, - Opts: opts, terminated: tch}, nil -} - -// Run all application objects -func (a *Application) Run(ctx context.Context) error { - if a.DevPasswd != "" { - log.Printf("[WARN] running in dev mode") - } - - go func() { - // shutdown on context cancellation - <-ctx.Done() - a.restSrv.Shutdown() - a.migratorSrv.Shutdown() - if a.devAuth != nil { - a.devAuth.Shutdown() - } - if e := a.dataService.Close(); e != nil { - log.Printf("[WARN] failed to close store, %s", e) - } - - }() - a.activateBackup(ctx) // runs in goroutine for each site - go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only - if a.Auth.Dev { - go a.devAuth.Run() // dev oauth2 server on :8084 - } - a.restSrv.Run(a.Port) - close(a.terminated) - return nil -} - -// Wait for application completion (termination) -func (a *Application) Wait() { - <-a.terminated -} - -// activateBackup runs background backups for each site -func (a *Application) activateBackup(ctx context.Context) { - for _, siteID := range a.Sites { - backup := migrator.AutoBackup{ - Exporter: a.exporter, - BackupLocation: a.BackupLocation, - SiteID: siteID, - KeepMax: a.MaxBackupFiles, - Duration: 24 * time.Hour, - } - go backup.Do(ctx) - } -} - -// makeDataStore creates store for all sites -func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) { - switch group.Type { - case "bolt": - if err = makeDirs(group.Bolt.Path); err != nil { - return nil, errors.Wrap(err, "failed to create bolt store") - } - sites := []engine.BoltSite{} - for _, site := range siteNames { - sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)}) - } - result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...) - case "mongo": - mgServer, e := makeMongo(mg) - if e != nil { - return result, errors.Wrap(e, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, mg.DB, "") - result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) - default: - return nil, errors.Errorf("unsupported store type %s", group.Type) - } - return result, errors.Wrap(err, "can't initialize data store") -} - -func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { - switch group.Type { - case "fs": - if err := makeDirs(group.FS.Path); err != nil { - return nil, err - } - return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil - case "mongo": - mgServer, err := makeMongo(mg) + p.CommandHandler = func(command flags.Commander, args []string) error { + setupLog(opts.Dbg) + err := command.Execute(args) if err != nil { - return nil, errors.Wrap(err, "failed to create mongo server") + log.Printf("[ERROR] failed with %+v", err) } - conn := mongo.NewConnection(mgServer, mg.DB, "") - return avatar.NewGridFS(conn, group.RszLmt), nil - } - return nil, errors.Errorf("unsupported avatar store type %s", group.Type) -} - -func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) { - switch group.Type { - case "shared": - return keys.NewStaticStore(sharedSecret), nil - default: - return nil, errors.Errorf("unsupported key store type %s", group.Type) - } -} - -func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { - switch group.Type { - case "mem": - return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), - cache.MaxKeys(group.Max.Items)) - case "mongo": - mgServer, err := makeMongo(mg) - if err != nil { - return nil, errors.Wrap(err, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, mg.DB, "cache") - return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), - cache.MaxKeys(group.Max.Items)) - } - return nil, errors.Errorf("unsupported cache type %s", group.Type) -} - -// mkdir -p for all dirs -func makeDirs(dirs ...string) error { - - // exists returns whether the given file or directory exists or not - exists := func(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return true, err + return err } - for _, dir := range dirs { - ex, err := exists(dir) - if err != nil { - return errors.Wrapf(err, "can't check directory status for %s", dir) - } - if !ex { - if e := os.MkdirAll(dir, 0700); e != nil { - return errors.Wrapf(err, "can't make directory %s", dir) - } - } - } - return nil -} - -func makeMongo(mg MongoGroup) (result *mongo.Server, err error) { - if mg.URL == "" { - return nil, errors.New("no mongo URL provided") - } - return mongo.NewServerWithURL(mg.URL, 10*time.Second) -} - -func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider { - - makeParams := func(cid, secret string) auth.Params { - return auth.Params{ - JwtService: jwtService, - AvatarProxy: avatarProxy, - RemarkURL: opts.RemarkURL, - Cid: cid, - Csecret: secret, - PermissionChecker: ds, - } - } - - providers := []auth.Provider{} - if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" { - providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC))) - } - if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" { - providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC))) - } - if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" { - providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC))) - } - if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" { - providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC))) - } - if opts.Auth.Dev { - providers = append(providers, auth.NewDev(makeParams("", ""))) - } - - if len(providers) == 0 { - log.Printf("[WARN] no auth providers defined") - } - return providers -} - -// resetEnv clears all sensitive env vars -func resetEnv(envs ...string) { - for _, env := range envs { - if err := os.Unsetenv(env); err != nil { - log.Printf("[WARN] can't unset env %s, %s", env, err) + if _, err := p.Parse(); err != nil { + if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { + os.Exit(0) + } else { + os.Exit(1) } } } diff --git a/backend/app/main_test.go b/backend/app/main_test.go index 436d082f..0ffe37a7 100644 --- a/backend/app/main_test.go +++ b/backend/app/main_test.go @@ -1,227 +1,48 @@ package main import ( - "context" - "fmt" "io/ioutil" - "log" "net/http" "os" - "strings" + "sync" "syscall" "testing" "time" - "github.com/globalsign/mgo" - "github.com/go-pkgz/mongo" - "github.com/jessevdk/go-flags" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestApplication(t *testing.T) { - app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts { - o.Port = 18080 - return o - }) +func TestMain(t *testing.T) { - go func() { _ = app.Run(ctx) }() - time.Sleep(100 * time.Millisecond) // let server start - - // send ping - resp, err := http.Get("http://localhost:18080/api/v1/ping") - require.Nil(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, "pong", string(body)) - - // add comment - resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json", - strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`)) - require.Nil(t, err) - assert.Equal(t, http.StatusCreated, resp.StatusCode) - body, _ = ioutil.ReadAll(resp.Body) - t.Log(string(body)) - - assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email") - - app.Wait() -} - -func TestApplicationDevMode(t *testing.T) { - app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts { - o.Port = 18085 - o.DevPasswd = "password" - o.Auth.Dev = true - return o - }) - - go func() { _ = app.Run(ctx) }() - time.Sleep(100 * time.Millisecond) // let server start - - assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider") - assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider") - // send ping - resp, err := http.Get("http://localhost:18085/api/v1/ping") - require.Nil(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, "pong", string(body)) - - app.Wait() -} - -func TestApplicationWithMongo(t *testing.T) { - - mongoURL := os.Getenv("MONGO_TEST") - if mongoURL == "" { - mongoURL = "mongodb://localhost:27017/test" - } - if mongoURL == "skip" { - t.Skip("skip mongo app test") - } - - opts := Opts{} - // prepare options - p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com", - "--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) - require.Nil(t, err) - opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" - opts.BackupLocation = "/tmp" - - // create app - app, err := New(opts) - require.Nil(t, err) - - defer func() { - s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second) - assert.NoError(t, err) - conn := mongo.NewConnection(s, "test_remark", "") - _ = conn.WithDB(func(dbase *mgo.Database) error { - assert.NoError(t, dbase.DropDatabase()) - return nil - }) - }() - - ctx, cancel := context.WithCancel(context.Background()) - go func() { - time.Sleep(5 * time.Second) - log.Print("[TEST] terminate app") - cancel() - }() - go func() { _ = app.Run(ctx) }() - time.Sleep(100 * time.Millisecond) // let server start - - // send ping - resp, err := http.Get("http://localhost:12345/api/v1/ping") - require.Nil(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, "pong", string(body)) - - app.Wait() -} - -func TestApplicationFailed(t *testing.T) { - opts := Opts{} - p := flags.NewParser(&opts, flags.Default) - - // RO bolt location - _, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", - "--store.bolt.path=/dev/null"}) - assert.Nil(t, err) - _, err = New(opts) - assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ - "open /dev/null/remark.db: not a directory") - t.Log(err) - - // RO backup location - opts = Opts{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp", - "--backup=/dev/null/not-writable"}) - assert.Nil(t, err) - _, err = New(opts) - assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory") - t.Log(err) - - // invalid url - opts = Opts{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"}) - assert.Nil(t, err) - _, err = New(opts) - assert.EqualError(t, err, "invalid remark42 url demo.remark42.com") - t.Log(err) - - opts = Opts{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"}) - assert.NotNil(t, err, "blah is invalid type") - - opts.Store.Type = "blah" - _, err = New(opts) - assert.EqualError(t, err, "unsupported store type blah") - t.Log(err) -} - -func TestApplicationShutdown(t *testing.T) { - app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts { - o.Port = 18090 - return o - }) - st := time.Now() - err := app.Run(ctx) - assert.Nil(t, err) - assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") - app.Wait() -} - -func TestApplicationMainSignal(t *testing.T) { - os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", - "--port=18100", "--url=https://demo.remark42.com"} + os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", + "--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg"} go func() { - time.Sleep(100 * time.Millisecond) + time.Sleep(500 * time.Millisecond) err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM) require.Nil(t, err) }() - st := time.Now() - main() - assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") -} -func prepApp(t *testing.T, duration time.Duration, fn func(o Opts) Opts) (*Application, context.Context) { - opts := Opts{} - // prepare options - p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"}) - require.Nil(t, err) - opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp" - opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port) - opts.Store.Bolt.Timeout = 10 * time.Second - opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" - opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid" - opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid" - opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid" - opts.BackupLocation = "/tmp" - opts = fn(opts) - - os.Remove(opts.Store.Bolt.Path + "/remark.db") - - // create app - app, err := New(opts) - require.Nil(t, err) - - ctx, cancel := context.WithCancel(context.Background()) + wg := sync.WaitGroup{} + wg.Add(1) go func() { - time.Sleep(duration) - log.Print("[TEST] terminate app") - cancel() + st := time.Now() + main() + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") + wg.Done() }() - return app, ctx + + time.Sleep(50 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:18202/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + wg.Wait() } diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 814a1323..c82caebf 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -60,7 +60,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment") return } - a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last")) + a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope)) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"id": id, "locator": locator}) } @@ -129,7 +129,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { } } - a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, "last")) + a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, lastCommentsScope)) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID}) } diff --git a/backend/app/rest/api/middleware.go b/backend/app/rest/api/middleware.go index 77185eb1..15ee1c7d 100644 --- a/backend/app/rest/api/middleware.go +++ b/backend/app/rest/api/middleware.go @@ -106,6 +106,7 @@ func Logger(ipFn func(ip string) string, flags ...LoggerFlag) func(http.Handler) if qun, err := url.QueryUnescape(q); err == nil { q = qun } + q = sanitizeQuery(q) remoteIP := strings.Split(r.RemoteAddr, ":")[0] if strings.HasPrefix(r.RemoteAddr, "[") { @@ -159,6 +160,27 @@ func getBodyAndUser(r *http.Request, flags []LoggerFlag) (body string, user stri return body, user } +func sanitizeQuery(u string) string { + out := []rune(u) + hide := []string{"password", "passwd", "secret", "credentials"} + for _, h := range hide { + if strings.Contains(strings.ToLower(u), h+"=") { + stPos := strings.Index(strings.ToLower(u), h+"=") + len(h) + 1 + fnPos := strings.Index(u[stPos:], "&") + if fnPos == -1 { + fnPos = len(u) + } else { + fnPos = stPos + fnPos + } + log.Print(stPos, fnPos) + for i := stPos; i < fnPos; i++ { + out[i] = rune('*') + } + } + } + return string(out) +} + func inLogFlags(f LoggerFlag, flags []LoggerFlag) bool { for _, flg := range flags { if (flg == LogAll && f != LogNone) || flg == f { diff --git a/backend/app/rest/api/middleware_test.go b/backend/app/rest/api/middleware_test.go index c2786057..a91f00ff 100644 --- a/backend/app/rest/api/middleware_test.go +++ b/backend/app/rest/api/middleware_test.go @@ -66,3 +66,19 @@ func TestMiddleware_GetBodyAndUser(t *testing.T) { assert.Equal(t, "", body) assert.Equal(t, ` - id1 "user1"`, user, "no user") } + +func TestMiddleware_sanitizeReqURL(t *testing.T) { + tbl := []struct { + in string + out string + }{ + {"", ""}, + {"/aa/bb?xyz=123", "/aa/bb?xyz=123"}, + {"/aa/bb?xyz=123&secret=asdfghjk", "/aa/bb?xyz=123&secret=********"}, + {"/aa/bb?xyz=123&secret=asdfghjk&key=val", "/aa/bb?xyz=123&secret=********&key=val"}, + {"/aa/bb?xyz=123&secret=asdfghjk&key=val&password=1234", "/aa/bb?xyz=123&secret=********&key=val&password=****"}, + } + for i, tt := range tbl { + assert.Equal(t, tt.out, sanitizeQuery(tt.in), "check #%d, %s", i, tt.in) + } +} diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 3010892a..51e5d331 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -145,7 +145,6 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/gzip") w.Header().Set("Content-Disposition", "attachment;filename="+exportFile) - w.WriteHeader(http.StatusOK) gzWriter := gzip.NewWriter(w) defer func() { if e := gzWriter.Close(); e != nil { @@ -154,6 +153,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { }() if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil { + log.Printf("[WARN] can't export, %+v", err) rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") return } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 57c18d2d..3c17dd71 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -60,6 +60,8 @@ type Rest struct { const hardBodyLimit = 1024 * 64 // limit size of body +const lastCommentsScope = "last" + type commentsWithInfo struct { Comments []store.Comment `json:"comments"` Info store.PostInfo `json:"info,omitempty"` @@ -123,7 +125,7 @@ func (s *Rest) routes() chi.Router { AllowedOrigins: []string{"*"}, AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"}, - ExposedHeaders: []string{"Link"}, + ExposedHeaders: []string{"Authorization"}, AllowCredentials: true, MaxAge: 300, }) diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 6f1edd0a..1a47a895 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -71,7 +71,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) { return } s.Cache.Flush(cache.Flusher(comment.Locator.SiteID). - Scopes(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)) + Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID)) render.Status(r, http.StatusCreated) render.JSON(w, r, &finalComment) @@ -120,7 +120,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last", user.ID)) + s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope, user.ID)) render.JSON(w, r, res) } diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index 6ee667bd..02460a68 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -114,23 +114,15 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) { limit = 0 } - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes("last") + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(lastCommentsScope) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, limit) if e != nil { return nil, e } comments = s.adminService.alterComments(comments, r) - // filter deleted from last comments view. Blocked marked as deleted and will sneak in without - filterDeleted := []store.Comment{} - for _, c := range comments { - if c.Deleted { - continue - } - filterDeleted = append(filterDeleted, c) - } - + filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted }) return encodeJSONWithHTML(filterDeleted) }) diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go index 31886ba1..a133cf1c 100644 --- a/backend/app/rest/api/rest_public_test.go +++ b/backend/app/rest/api/rest_public_test.go @@ -238,6 +238,7 @@ func TestRest_Last(t *testing.T) { err = json.Unmarshal([]byte(res), &comments) assert.Nil(t, err) assert.Equal(t, 2, len(comments), "should have 2 comments") + t.Logf("%+v", comments) } func TestRest_FindUserComments(t *testing.T) { diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 43a2f76c..cf1f7bc0 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -72,6 +72,21 @@ func TestRest_Shutdown(t *testing.T) { assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms") } +func TestRest_filterComments(t *testing.T) { + user := store.User{ID: "user1", Name: "user name 1"} + c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)} + c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)} + c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)} + + r := filterComments([]store.Comment{c1, c2, c3}, func(c store.Comment) bool { + return c.Text == "test test #1" || c.Text == "test test #3" + }) + assert.Equal(t, 2, len(r), "one comment filtered") +} + func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go index 39ef0fb6..da0562cf 100644 --- a/backend/app/rest/api/rss.go +++ b/backend/app/rest/api/rss.go @@ -67,7 +67,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss for site %s", siteID) - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last") + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, maxRssItems) if e != nil { @@ -100,7 +100,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID) - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last") + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope) data, err := s.Cache.Get(key, func() (res []byte, e error) { comments, e := s.DataService.Last(siteID, maxLastCommentsReply) if e != nil { diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index e66c2131..a77446a0 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -135,22 +135,27 @@ func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) boo return false } + log.Printf("[DEBUG] dev user auth") s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) if len(s) != 2 { + log.Printf("[WARN] dev user auth failed, incorrect auth header %s", r.Header.Get("Authorization")) return false } b, err := base64.StdEncoding.DecodeString(s[1]) if err != nil { + log.Printf("[WARN] dev user auth failed, failed to decode %s, %s", s[1], err) return false } pair := strings.SplitN(string(b), ":", 2) if len(pair) != 2 { + log.Printf("[WARN] dev user auth failed, failed to split %s", string(b)) return false } if pair[0] != "dev" || pair[1] != a.DevPasswd { + log.Printf("[WARN] dev user auth failed, user/passwd mismatch %+v", pair) return false } diff --git a/backend/app/rest/proxy/avatar_test.go b/backend/app/rest/proxy/avatar_test.go index eb84bb75..53eba9a6 100644 --- a/backend/app/rest/proxy/avatar_test.go +++ b/backend/app/rest/proxy/avatar_test.go @@ -66,7 +66,7 @@ func TestAvatar_PutFailed(t *testing.T) { _, err := p.Put(u) assert.EqualError(t, err, "no picture for user1") - u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:12345/avater/pic"} + u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:22345/avater/pic"} _, err = p.Put(u) require.Error(t, err) assert.Contains(t, err.Error(), "connect: connection refused") diff --git a/backend/scripts/backup.sh b/backend/scripts/backup.sh new file mode 100644 index 00000000..4915df17 --- /dev/null +++ b/backend/scripts/backup.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +/srv/remark42 backup $@ diff --git a/backend/scripts/create-backup.sh b/backend/scripts/create-backup.sh deleted file mode 100755 index 38880440..00000000 --- a/backend/scripts/create-backup.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -# this scrips makes a backup file to /srv/var/userbackup--.gz -set -e -BACKUP_PATH=${BACKUP_PATH:-./var} -backup_file=${BACKUP_PATH}/userbackup-${1}-$(date +%s).gz -echo "make backup file for site $1 to $backup_file" -curl "http://127.0.0.1:8081/api/v1/admin/export?site=${1}&secret=${SECRET}" > ${backup_file} -echo "created backup ${backup_file}" \ No newline at end of file diff --git a/backend/scripts/import-disqus.sh b/backend/scripts/import-disqus.sh deleted file mode 100644 index 7b35e49a..00000000 --- a/backend/scripts/import-disqus.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -e -echo "import disqus file $1 to site $2" -curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=disqus&secret=${SECRET}" -echo "import completed" \ No newline at end of file diff --git a/backend/scripts/import-wordpress.sh b/backend/scripts/import-wordpress.sh deleted file mode 100755 index 8e60dabf..00000000 --- a/backend/scripts/import-wordpress.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -e -echo "import wordpress file $1 to site $2" -curl -X POST -H "Content-Type: application/xml" --data-binary @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=wordpress&secret=${SECRET}" -echo "import completed" \ No newline at end of file diff --git a/backend/scripts/import.sh b/backend/scripts/import.sh new file mode 100644 index 00000000..aa5d5492 --- /dev/null +++ b/backend/scripts/import.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +/srv/remark42 import $@ diff --git a/backend/scripts/migrate-data.sh b/backend/scripts/migrate-data.sh deleted file mode 100755 index 7397a0a4..00000000 --- a/backend/scripts/migrate-data.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# this scrips making a backup file to /tmp/export-remark.gz and loading it back -# useful to migrate data schema in case if new version of data store incomaptible with the stored comments. -set -e -echo "make backup file for site $1" -curl "http://127.0.0.1:8081/api/v1/admin/export?site=${1}&secret=${SECRET}" > /tmp/export-remark.gz - -BOLTDB_PATH=${BOLTDB_PATH:-./var} -BACKUP_PATH=${BACKUP_PATH:-./var} - -cp ${BOLTDB_PATH}/${1}.db ${BACKUP_PATH}/${1}-$(date +%s).db - -echo "import backup to site $1" -echo "unpack /tmp/export-remark.gz" -gunzip -c /tmp/export-remark.gz >/tmp/backup.remark -ls -laH /tmp/backup.remark - -echo "export to site $1" -curl -X POST -H "Content-Type: application/json" --data-binary @/tmp/backup.remark "http://127.0.0.1:8081/api/v1/admin/import?site=${1}&provider=native&secret=${SECRET}" - -rm /tmp/backup.remark -rm /tmp/export-remark.gz - -echo "migration completed" \ No newline at end of file diff --git a/backend/scripts/restore-backup.sh b/backend/scripts/restore-backup.sh deleted file mode 100755 index a50b0104..00000000 --- a/backend/scripts/restore-backup.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -set -e -echo "restore backup file $1 to site $2" -BACKUP_PATH=${BACKUP_PATH:-./var} -echo "unpack $1" -gunzip -c ${BACKUP_PATH}/$1 >/tmp/backup.remark - -echo "source file info" -ls -la /tmp/backup.remark - -curl -X POST -H "Content-Type: application/json" --data-binary @/tmp/backup.remark "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=native&secret=${SECRET}" -rm /tmp/backup.remark - -echo "backup restored" \ No newline at end of file diff --git a/backend/scripts/restore.sh b/backend/scripts/restore.sh new file mode 100644 index 00000000..b308abb1 --- /dev/null +++ b/backend/scripts/restore.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +/srv/remark42 restore $@ diff --git a/compose-dev-backend.yml b/compose-dev-backend.yml index 8af2392b..6ad99f6e 100644 --- a/compose-dev-backend.yml +++ b/compose-dev-backend.yml @@ -35,7 +35,6 @@ services: - "8084:8084" # local oauth2 server environment: - - USER # preset environment, UID on the host machine, i.e `id -u` - REMARK_URL=http://127.0.0.1:8080 - SECRET=12345 - STORE_BOLT_PATH=/srv/var/db @@ -46,5 +45,3 @@ services: - ADMIN=dev_user # set admin flag for default user on local ouath2 volumes: - ./var:/srv/var - - command: /srv/start.sh diff --git a/compose-dev-frontend.yml b/compose-dev-frontend.yml index 7d57674a..e1533c9c 100644 --- a/compose-dev-frontend.yml +++ b/compose-dev-frontend.yml @@ -28,7 +28,6 @@ services: - "8084:8084" # local oauth2 server environment: - - USER # preset environment, UID on the host machine, i.e `id -u` - REMARK_URL=http://127.0.0.1:8080 - SECRET=12345 - STORE_BOLT_PATH=/srv/var/db @@ -39,5 +38,3 @@ services: - ADMIN=dev_user # set admin flag for default user on local ouath2 volumes: - ./var:/srv/var - - command: /srv/start.sh diff --git a/docker-compose.yml b/docker-compose.yml index 9548e384..cce74474 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,6 @@ services: # - "80:8080" environment: - - USER - REMARK_URL - SECRET - STORE_BOLT_PATH=/srv/var/db @@ -36,5 +35,3 @@ services: # - DEV_PASSWD=password # development mode, be careful! volumes: - ./var:/srv/var - - command: /srv/start.sh diff --git a/start.sh b/entrypoint.sh similarity index 63% rename from start.sh rename to entrypoint.sh index c8a0c1d3..82867cb5 100755 --- a/start.sh +++ b/entrypoint.sh @@ -1,19 +1,11 @@ #!/bin/sh - echo "prepare environment" - # replace BASE_URL constant by REMARK_URL sed -i "s|https://demo.remark42.com|${REMARK_URL}|g" /srv/web/*.js # remove devtools attach helper. TODO: move to webpack loader sed -i "/REMOVE-START/,/REMOVE-END/d" /srv/web/iframe.html +chown -R app:app /srv/var 2>/dev/null + echo "start remark42 server" - -if [ -z "$USER" ] ; then \ - echo "No USER defined, runs under root!" - exec /srv/remark - -else - echo "runs under ${USER}" - /sbin/su-exec ${USER} /srv/remark -fi +/sbin/su-exec app /srv/remark42 $@