mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature/cmd (#195)

Browse Source 
* support flags commands, move to cmd

* fix target name

* test for happy path importer

* add export cmd

* fix wrong import, lint warns

* increase test timeout

* add sellp to allow main test server to start

* implement all cmds

* handle backup/restore errors

* fix import status check, hide secret from logs

* backup cmd err tests

* randimize test port

* avoid dup code in Last controller

* add target to make all bin archives

* remove container in make

* add smiple scripts to simplify commands, update readme

* add docs on dockerless, enforce app user

* add restore info

* move last to lastCommentsScope const
This commit is contained in:
Umputun
2018-08-26 21:45:04 -05:00
committed by GitHub
parent 419d2ae75e
commit ba6bab5b91
44 changed files with 1400 additions and 756 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+2
View File
@@ -19,3 +19,5 @@ debug
debug.test
*.prof
*.test
/bin/
remark42
.gitignore
+2
View File
@@ -16,3 +16,5 @@ debug.test
/rest-client.env.json
.DS_Store
.mongo
remark42
/bin/
Dockerfile
+10 -7
View File
@@ -58,7 +58,7 @@ RUN \
echo "runs outside of drone" && version="local"; \
else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
echo "version=$version" && \
go build -o remark -ldflags "-X main.revision=${version} -s -w" ./app
go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
FROM node:10.6-alpine as build-frontend-deps
@@ -88,16 +88,19 @@ FROM umputun/baseimage:app-latest
WORKDIR /srv
ADD backend/scripts/*.sh /srv/
ADD start.sh /srv/start.sh
RUN chmod +x /srv/*.sh
ADD entrypoint.sh /entrypoint.sh
ADD backend/scripts/backup.sh /usr/local/bin/backup
ADD backend/scripts/restore.sh /usr/local/bin/restore
ADD backend/scripts/import.sh /usr/local/bin/import
RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark /srv/
COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
COPY --from=build-frontend /srv/web/public/ /srv/web
RUN chown -R app:app /srv
RUN ln -s /srv/remark42 /usr/bin/remark42
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
CMD ["/srv/start.sh"]
ENTRYPOINT ["/init.sh"]
CMD ["server"]
ENTRYPOINT ["/entrypoint.sh"]
Dockerfile.artifacts
+2 -1
View File
@@ -59,7 +59,7 @@ RUN \
tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
zip remark${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
# upload to github
RUN \
@@ -84,4 +84,5 @@ RUN \
FROM alpine
COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/
RUN ls -la /artifacts/*
CMD ["sleep", "100"]
Makefile
+15
View File
@@ -3,9 +3,24 @@ ARCH=amd64
bin:
docker build -f Dockerfile.artifacts -t remark42.bin .
- @docker rm -f remark42.bin 2>/dev/null || exit 0
docker run -d --name=remark42.bin remark42.bin
docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42
docker rm -f remark42.bin
docker:
docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
deploy:
docker build -f Dockerfile.artifacts -t remark42.bin .
- @docker rm -f remark42.bin 2>/dev/null || exit 0
- @mkdir -p bin
docker run -d --name=remark42.bin remark42.bin
docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz
docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
docker rm -f remark42.bin
README.md
+27 -17
View File
@@ -24,6 +24,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
- [Install](#install)
- [Backend](#backend)
- [With Docker](#with-docker)
- [Without docker](#without-docker)
- [Parameters](#parameters)
- [Required parameters](#required-parameters)
- [Register oauth2 providers](#register-oauth2-providers)
@@ -35,8 +37,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
- [Initial import from WordPress](#initial-import-from-wordpress)
- [Backup and restore](#backup-and-restore)
- [Automatic backups](#automatic-backups)
- [Schema migration](#schema-migration)
- [Manual backup](#manual-backup)
- [Restore from backup](#restore-from-backup)
- [Backup format](#backup-format)
- [Admin users](#admin-users)
- [Setup on your website](#setup-on-your-website)
@@ -62,11 +64,21 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
### Backend
#### With Docker
_this is the recommended way to run remark42_
* copy provided `docker-compose.yml` and customize for your needs
* prepare user id for container `` export USER=`id -u $USER` ``
* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
* alternatively compile from sources - `docker-compose build && docker-compose up -d`
* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
* alternatively compile from the sources - `docker-compose build && docker-compose up -d`
#### Without docker
* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
* unpack with `gunzip` (linux, mac os) or with `zip` (windows)
* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
#### Parameters
@@ -139,7 +151,6 @@ services:
- SECRET=abcd-123456-xyz-$%^& # secret key
- AUTH_GITHUB_CID=12345667890 # oauth2 client ID
- AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret
- USER=1001 # UID on the host machine, i.e `id -u`
volumes:
- ./var:/srv/var # persistent volume to store all remark42 data
```
@@ -200,13 +211,13 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co
1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
3. Run import command - `docker-compose exec remark42 /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
3. Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}`
#### Initial import from WordPress
1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments.
2. Move this file to your remark42 host within `./var`
3. Run import command - `docker-compose exec remark42 /srv/import-wordpress.sh <wordpress-export-name>.xml <your site id>`
3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
#### Backup and restore
@@ -215,20 +226,19 @@ Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./
For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container:
`docker-compose exec remark42 /srv/restore-backup.sh {backup-filename.gz} {your site id}`
##### Schema migration
One special case for backup/restore is schema migration. Some versions or remark42 may extend or change the schema
and for such upgrades migration required. Provided migration script `scripts/migrate-data.sh` makes a fresh backup and then loads it back to your remark42 instance.
`docker-compose exec remark42 /srv/migrate-data.sh {your site id}`
`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}`
##### Manual backup
In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz`
In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default.
`docker-compose exec remark42 /srv/create-backup.sh {your site id}`
`docker exec -it remark42 backup -s {your site id}`
##### Restore from backup
Restore will clean all comments first and then will processed with complete import from a given file.
`docker exec -it remark42 restore -f {backup file name} -s {your site id}`
##### Backup format
backend/app/cmd/backup.go
+85
View File
@@ -0,0 +1,85 @@
package cmd
import (
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"time"
"github.com/pkg/errors"
)
// BackupCommand set of flags and command for export
// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. /) BACKUP_PATH ignored.
type BackupCommand struct {
ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
Site string `long:"site" env:"SITE" default:"remark" description:"site name"`
SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"`
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"`
}
// Execute runs export with ExportCommand parameters, entry point for "export" command
func (ec *BackupCommand) Execute(args []string) error {
log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
resetEnv("SECRET")
fp := fileParser{site: ec.Site, path: ec.ExportPath, file: ec.ExportFile}
fname, err := fp.parse(time.Now())
if err != nil {
return err
}
log.Printf("[DEBUG] export file %s", fname)
// prepare http client
client := http.Client{}
exportURL := fmt.Sprintf("%s/api/v1/admin/export?site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret)
req, err := http.NewRequest(http.MethodGet, exportURL, nil)
if err != nil {
return errors.Wrapf(err, "can't make export request for %s", exportURL)
}
ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
defer cancel()
// get with timeout
resp, err := client.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrapf(err, "request failed for %s", exportURL)
}
defer func() {
if err = resp.Body.Close(); err != nil {
log.Printf("[WARN] failed to close response, %s", err)
}
}()
if resp.StatusCode >= 300 {
body, e := ioutil.ReadAll(resp.Body)
if e != nil {
body = []byte("")
}
return errors.Errorf("error response %q, %s", resp.Status, body)
}
fh, err := os.Create(fname)
if err != nil {
return errors.Wrapf(err, "can't create backup file %s", fname)
}
defer func() {
if err = fh.Close(); err != nil {
log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err)
}
}()
if _, err = io.Copy(fh, resp.Body); err != nil {
return errors.Wrapf(err, "failed to write backup file %s", fname)
}
log.Printf("[INFO] export completed, file %s", fname)
return nil
}
backend/app/cmd/backup_test.go
+71
View File
@@ -0,0 +1,71 @@
package cmd
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
flags "github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBackup_Execute(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
assert.Equal(t, "GET", r.Method)
fmt.Fprint(w, "blah\nblah2\n12345678\n")
}))
defer ts.Close()
cmd := BackupCommand{}
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp",
"--file={{.SITE}}-test.export", "--url=" + ts.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
defer os.Remove("/tmp/remark-test.export")
data, err := ioutil.ReadFile("/tmp/remark-test.export")
require.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
}
func TestBackup_ExecuteFailedStatus(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
assert.Equal(t, "GET", r.Method)
w.WriteHeader(400)
fmt.Fprint(w, "some error")
}))
defer ts.Close()
cmd := BackupCommand{}
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp",
"--file={{.SITE}}-test.export", "--url=" + ts.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.EqualError(t, err, `error response "400 Bad Request", some error`)
}
func TestBackup_ExecuteFailedWrite(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
assert.Equal(t, "GET", r.Method)
fmt.Fprint(w, "blah\nblah2\n12345678\n")
}))
defer ts.Close()
cmd := BackupCommand{}
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp",
"--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--url=" + ts.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
}
backend/app/cmd/cmd.go
+66
View File
@@ -0,0 +1,66 @@
// Package cmd has all top-level commands dispatched by main's flag.Parse
// The entry point of each command is Execute function
package cmd
import (
"bytes"
"log"
"os"
"path/filepath"
"strings"
"text/template"
"time"
"github.com/pkg/errors"
)
// fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
type fileParser struct {
site string
file string
path string
}
// parse apply template and also concat path and file. In case if file contains path separator path will be ignored
func (p *fileParser) parse(now time.Time) (string, error) {
fileTemplate := struct {
YYYYMMDD string
YYYY string
YYYYMM string
MM string
DD string
TS string
UNIX int64
SITE string
}{
YYYYMMDD: now.Format("20060102"),
YYYY: now.Format("2006"),
YYYYMM: now.Format("200601"),
MM: now.Format("01"),
DD: now.Format("02"),
UNIX: now.Unix(),
SITE: p.site,
TS: now.Format("20060102T150405"),
}
bb := bytes.Buffer{}
fname := p.file
if !strings.Contains(p.file, string(filepath.Separator)) {
fname = filepath.Join(p.path, p.file)
}
if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil {
return "", errors.Wrapf(err, "failed to parse %q", fname)
}
return bb.String(), nil
}
// resetEnv clears sensitive env vars
func resetEnv(envs ...string) {
for _, env := range envs {
if err := os.Unsetenv(env); err != nil {
log.Printf("[WARN] can't unset env %s, %s", env, err)
}
}
}
backend/app/cmd/cmd_test.go
+39
View File
@@ -0,0 +1,39 @@
package cmd
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestExport_ParseFileName(t *testing.T) {
tbl := []struct {
p fileParser
res string
err bool
}{
{fileParser{}, "", false},
{fileParser{path: "/tmp/blah", file: "fname.gz"}, "/tmp/blah/fname.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYYMMDD}}.gz"},
"/tmp/blah/fname-remark-20180821.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"},
"/tmp/blah/fname-remark-2018-08.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"},
"/tmp/fname-remark-2018-08.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.TS}}.gz"},
"/tmp/fname-remark-20180821T212615.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.XXX}}-{{.YYYY}}-{{.MM}}.gz"},
"", true},
}
now := time.Date(2018, 8, 21, 21, 26, 15, 0, time.UTC)
for i, tt := range tbl {
r, err := tt.p.parse(now)
if tt.err {
assert.NotNil(t, err)
continue
}
assert.Equal(t, tt.res, r, "check #%d", i)
}
}
backend/app/cmd/import.go
+85
View File
@@ -0,0 +1,85 @@
package cmd
import (
"compress/gzip"
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"time"
"github.com/pkg/errors"
)
// ImportCommand set of flags and command for import
type ImportCommand struct {
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"`
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"`
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"`
}
// Execute runs import with ImportCommand parameters, entry point for "import" command
func (ic *ImportCommand) Execute(args []string) error {
log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site)
resetEnv("SECRET")
reader, err := ic.reader(ic.InputFile)
if err != nil {
return errors.Wrapf(err, "can't open import file %s", ic.InputFile)
}
client := http.Client{}
importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s", ic.URL, ic.Site, ic.Provider, ic.SharedSecret)
req, err := http.NewRequest(http.MethodPost, importURL, reader)
if err != nil {
return errors.Wrapf(err, "can't make import request for %s", importURL)
}
ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout)
defer cancel()
resp, err := client.Do(req.WithContext(ctx)) // closes reader
if err != nil {
return errors.Wrapf(err, "request failed for %s", importURL)
}
defer func() {
if err = resp.Body.Close(); err != nil {
log.Printf("[WARN] failed to close response, %s", err)
}
}()
if resp.StatusCode >= 300 {
return errors.Errorf("error response %s (%d)", resp.Status, resp.StatusCode)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "can't get response from importer")
}
log.Printf("[INFO] import completed, status=%d, %s", resp.StatusCode, string(body))
return nil
}
// reader returns reader for file. For .gz file wraps with gunzip
func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
inpFile, err := os.Open(inp)
if err != nil {
return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
}
reader = inpFile
if strings.HasSuffix(ic.InputFile, ".gz") {
if reader, err = gzip.NewReader(inpFile); err != nil {
return nil, errors.Wrap(err, "can't make gz reader")
}
}
return reader, nil
}
backend/app/cmd/import_test.go
+112
View File
@@ -0,0 +1,112 @@
package cmd
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
flags "github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestImport_Execute(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
body, err := ioutil.ReadAll(r.Body)
assert.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
fmt.Fprintln(w, "some response")
fmt.Fprintln(w, string(body))
}))
defer ts.Close()
cmd := ImportCommand{}
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
cmd = ImportCommand{}
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt.gz", "--url=" + ts.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
}
func TestImport_ExecuteFailed(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
fmt.Fprintln(w, "some response")
}))
defer ts.Close()
cmd := ImportCommand{}
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import-no.txt", "--url=" + ts.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
t.Log(err)
assert.NotNil(t, err, "fail on no such file")
assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
cmd = ImportCommand{}
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt",
"--url=http://127.0.0.1:12345"})
require.Nil(t, err)
err = cmd.Execute(nil)
t.Log(err)
assert.NotNil(t, err, "fail on connection refused")
assert.True(t, strings.Contains(err.Error(), "connection refused"))
ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("%+v", r)
w.WriteHeader(400)
fmt.Fprintln(w, "some response with 400")
}))
defer ts2.Close()
cmd = ImportCommand{}
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts2.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
t.Log(err)
assert.NotNil(t, err)
}
func TestImport_ExecuteTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
body, err := ioutil.ReadAll(r.Body)
assert.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
time.Sleep(500 * time.Millisecond)
fmt.Fprintln(w, "some response")
fmt.Fprintln(w, string(body))
}))
defer ts.Close()
cmd := ImportCommand{}
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt",
"--url=" + ts.URL, "--timeout=300ms"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NotNil(t, err)
assert.True(t, strings.Contains(err.Error(), "deadline exceeded"))
}
backend/app/cmd/restore.go
+39
View File
@@ -0,0 +1,39 @@
package cmd
import (
"log"
"time"
)
// RestoreCommand set of flags and command for restore from backup
type RestoreCommand struct {
ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"`
Site string `long:"site" env:"SITE" default:"remark" description:"site name"`
SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"`
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"`
}
// Execute runs import with RestoreCommand parameters, entry point for "restore" command
// uses ImportCommand with constructed full file name
func (rc *RestoreCommand) Execute(args []string) error {
log.Printf("[INFO] restore %s, site %s", rc.ImportFile, rc.Site)
resetEnv("SECRET")
fp := fileParser{site: rc.Site, path: rc.ImportPath, file: rc.ImportFile}
fname, err := fp.parse(time.Now())
if err != nil {
return err
}
importer := ImportCommand{
InputFile: fname,
Site: rc.Site,
Provider: "native",
SharedSecret: rc.SharedSecret,
Timeout: rc.Timeout,
URL: rc.URL,
}
return importer.Execute(args)
}
backend/app/cmd/restore_test.go
+37
View File
@@ -0,0 +1,37 @@
package cmd
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
flags "github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRestore_Execute(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
assert.Equal(t, "native", r.URL.Query().Get("provider"))
body, err := ioutil.ReadAll(r.Body)
assert.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
fmt.Fprintln(w, "some response")
fmt.Fprintln(w, string(body))
}))
defer ts.Close()
cmd := RestoreCommand{}
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=testdata", "--file=import.txt",
"--url=" + ts.URL})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
}
backend/app/cmd/server.go
+446
View File
@@ -0,0 +1,446 @@
package cmd
import (
"context"
"fmt"
"log"
"net/url"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/coreos/bbolt"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest/api"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
)
// ServerOpts with command line flags and env
type ServerOpts struct {
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"`
Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"`
Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"`
Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"`
DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
// Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
Auth struct {
TTL struct {
JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
}
// AuthGroup defines options group for auth params
type AuthGroup struct {
CID string `long:"cid" env:"CID" description:"OAuth client ID"`
CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
}
// StoreGroup defines options group for store params
type StoreGroup struct {
Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
Bolt struct {
Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
}
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
}
// CacheGroup defines options group for cache params
type CacheGroup struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"`
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
} `group:"max" namespace:"max" env-namespace:"MAX"`
}
// MongoGroup holds all mongo params, used by store, avatar and cache
type MongoGroup struct {
URL string `long:"url" env:"URL" description:"mongo url"`
DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
}
// KeyGroup defines options group for key params
type KeyGroup struct {
Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"`
}
// Revision sets from main
var Revision = "unknown"
// serverApp holds all active objects
type serverApp struct {
*ServerOpts
restSrv *api.Rest
migratorSrv *api.Migrator
exporter migrator.Exporter
devAuth *auth.DevAuthServer
dataService *service.DataStore
terminated chan struct{}
}
// Execute is the entry point for "server" command, called by flag parser
func (s *ServerOpts) Execute(args []string) error {
log.Print("[INFO] start remark42 server")
resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
ctx, cancel := context.WithCancel(context.Background())
go func() { // catch signal and invoke graceful termination
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
<-stop
log.Print("[WARN] interrupt signal")
cancel()
}()
app, err := newServerApp(s)
if err != nil {
log.Fatalf("[ERROR] failed to setup application, %+v", err)
}
if err = app.run(ctx); err != nil {
log.Printf("[INFO] remark terminated with error %+v", err)
return err
}
log.Printf("[INFO] remark terminated")
return nil
}
// newServerApp prepares application and return it with all active parts
// doesn't start anything
func newServerApp(opts *ServerOpts) (*serverApp, error) {
if err := makeDirs(opts.BackupLocation); err != nil {
return nil, err
}
if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") {
return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
}
storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites)
if err != nil {
return nil, err
}
keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret)
if err != nil {
return nil, err
}
dataService := &service.DataStore{
Interface: storeEngine,
EditDuration: opts.EditDuration,
KeyStore: keyStore,
MaxCommentSize: opts.MaxCommentSize,
Admins: opts.Admins,
}
loadingCache, err := makeCache(opts.Cache, opts.Mongo)
if err != nil {
return nil, err
}
// token TTL is 5 minutes, inactivity interval 7+ days by default
jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie)
avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo)
if err != nil {
return nil, errors.Wrap(err, "failed to make avatar store")
}
avatarProxy := &proxy.Avatar{
Store: avatarStore,
RoutePath: "/api/v1/avatar",
RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
}
exporter := &migrator.Remark{DataStore: dataService}
migr := &api.Migrator{
Version: Revision,
Cache: loadingCache,
NativeImporter: &migrator.Remark{DataStore: dataService},
DisqusImporter: &migrator.Disqus{DataStore: dataService},
WordPressImporter: &migrator.WordPress{DataStore: dataService},
NativeExported: &migrator.Remark{DataStore: dataService},
KeyStore: keyStore,
}
authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts)
imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}
commentFormatter := store.NewCommentFormatter(imgProxy)
srv := &api.Rest{
Version: Revision,
DataService: dataService,
Exporter: exporter,
WebRoot: opts.WebRoot,
RemarkURL: opts.RemarkURL,
ImageProxy: imgProxy,
CommentFormatter: commentFormatter,
AvatarProxy: avatarProxy,
ReadOnlyAge: opts.ReadOnlyAge,
SharedSecret: opts.SharedSecret,
Authenticator: auth.Authenticator{
JWTService: jwtService,
AdminEmail: opts.AdminEmail,
Providers: authProviders,
DevPasswd: opts.DevPasswd,
PermissionChecker: dataService,
},
Cache: loadingCache,
}
// no admin email, use admin@domain
if srv.Authenticator.AdminEmail == "" {
if u, err := url.Parse(opts.RemarkURL); err == nil {
srv.Authenticator.AdminEmail = "admin@" + u.Host
}
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore
var devAuth *auth.DevAuthServer
if opts.Auth.Dev {
devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]}
}
tch := make(chan struct{})
return &serverApp{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService,
ServerOpts: opts, terminated: tch}, nil
}
// Run all application objects
func (a *serverApp) run(ctx context.Context) error {
if a.DevPasswd != "" {
log.Printf("[WARN] running in dev mode")
}
go func() {
// shutdown on context cancellation
<-ctx.Done()
a.restSrv.Shutdown()
a.migratorSrv.Shutdown()
if a.devAuth != nil {
a.devAuth.Shutdown()
}
if e := a.dataService.Close(); e != nil {
log.Printf("[WARN] failed to close store, %s", e)
}
}()
a.activateBackup(ctx) // runs in goroutine for each site
go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only
if a.Auth.Dev {
go a.devAuth.Run() // dev oauth2 server on :8084
}
a.restSrv.Run(a.Port)
close(a.terminated)
return nil
}
// Wait for application completion (termination)
func (a *serverApp) Wait() {
<-a.terminated
}
// activateBackup runs background backups for each site
func (a *serverApp) activateBackup(ctx context.Context) {
for _, siteID := range a.Sites {
backup := migrator.AutoBackup{
Exporter: a.exporter,
BackupLocation: a.BackupLocation,
SiteID: siteID,
KeepMax: a.MaxBackupFiles,
Duration: 24 * time.Hour,
}
go backup.Do(ctx)
}
}
// makeDataStore creates store for all sites
func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) {
switch group.Type {
case "bolt":
if err = makeDirs(group.Bolt.Path); err != nil {
return nil, errors.Wrap(err, "failed to create bolt store")
}
sites := []engine.BoltSite{}
for _, site := range siteNames {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
case "mongo":
mgServer, e := makeMongo(mg)
if e != nil {
return result, errors.Wrap(e, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "")
result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
default:
return nil, errors.Errorf("unsupported store type %s", group.Type)
}
return result, errors.Wrap(err, "can't initialize data store")
}
func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) {
switch group.Type {
case "fs":
if err := makeDirs(group.FS.Path); err != nil {
return nil, err
}
return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil
case "mongo":
mgServer, err := makeMongo(mg)
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "")
return avatar.NewGridFS(conn, group.RszLmt), nil
}
return nil, errors.Errorf("unsupported avatar store type %s", group.Type)
}
func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) {
switch group.Type {
case "shared":
return keys.NewStaticStore(sharedSecret), nil
default:
return nil, errors.Errorf("unsupported key store type %s", group.Type)
}
}
func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) {
switch group.Type {
case "mem":
return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
cache.MaxKeys(group.Max.Items))
case "mongo":
mgServer, err := makeMongo(mg)
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "cache")
return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
cache.MaxKeys(group.Max.Items))
}
return nil, errors.Errorf("unsupported cache type %s", group.Type)
}
// mkdir -p for all dirs
func makeDirs(dirs ...string) error {
// exists returns whether the given file or directory exists or not
exists := func(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
for _, dir := range dirs {
ex, err := exists(dir)
if err != nil {
return errors.Wrapf(err, "can't check directory status for %s", dir)
}
if !ex {
if e := os.MkdirAll(dir, 0700); e != nil {
return errors.Wrapf(err, "can't make directory %s", dir)
}
}
}
return nil
}
func makeMongo(mg MongoGroup) (result *mongo.Server, err error) {
if mg.URL == "" {
return nil, errors.New("no mongo URL provided")
}
return mongo.NewServerWithURL(mg.URL, 10*time.Second)
}
func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts *ServerOpts) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
return auth.Params{
JwtService: jwtService,
AvatarProxy: avatarProxy,
RemarkURL: opts.RemarkURL,
Cid: cid,
Csecret: secret,
PermissionChecker: ds,
}
}
providers := []auth.Provider{}
if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC)))
}
if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC)))
}
if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC)))
}
if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" {
providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC)))
}
if opts.Auth.Dev {
providers = append(providers, auth.NewDev(makeParams("", "")))
}
if len(providers) == 0 {
log.Printf("[WARN] no auth providers defined")
}
return providers
}
backend/app/cmd/server_test.go
+232
View File
@@ -0,0 +1,232 @@
package cmd
import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"syscall"
"testing"
"time"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestServerApp(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts {
o.Port = 18080
return o
})
go func() { _ = app.run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:18080/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
// add comment
resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json",
strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
require.Nil(t, err)
assert.Equal(t, http.StatusCreated, resp.StatusCode)
body, _ = ioutil.ReadAll(resp.Body)
t.Log(string(body))
assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email")
app.Wait()
}
func TestServerApp_DevMode(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts {
o.Port = 18085
o.DevPasswd = "password"
o.Auth.Dev = true
return o
})
go func() { _ = app.run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider")
assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider")
// send ping
resp, err := http.Get("http://localhost:18085/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
app.Wait()
}
func TestServerApp_WithMongo(t *testing.T) {
mongoURL := os.Getenv("MONGO_TEST")
if mongoURL == "" {
mongoURL = "mongodb://localhost:27017/test"
}
if mongoURL == "skip" {
t.Skip("skip mongo app test")
}
opts := ServerOpts{}
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com",
"--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"})
require.Nil(t, err)
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.BackupLocation = "/tmp"
// create app
app, err := newServerApp(&opts)
require.Nil(t, err)
defer func() {
s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
assert.NoError(t, err)
conn := mongo.NewConnection(s, "test_remark", "")
_ = conn.WithDB(func(dbase *mgo.Database) error {
assert.NoError(t, dbase.DropDatabase())
return nil
})
}()
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(5 * time.Second)
log.Print("[TEST] terminate app")
cancel()
}()
go func() { _ = app.run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:12345/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
app.Wait()
}
func TestServerApp_Failed(t *testing.T) {
opts := ServerOpts{}
p := flags.NewParser(&opts, flags.Default)
// RO bolt location
_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp",
"--store.bolt.path=/dev/null"})
assert.Nil(t, err)
_, err = newServerApp(&opts)
assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
"open /dev/null/remark.db: not a directory")
t.Log(err)
// RO backup location
opts = ServerOpts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
"--backup=/dev/null/not-writable"})
assert.Nil(t, err)
_, err = newServerApp(&opts)
assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
t.Log(err)
// invalid url
opts = ServerOpts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"})
assert.Nil(t, err)
_, err = newServerApp(&opts)
assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
t.Log(err)
opts = ServerOpts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"})
assert.NotNil(t, err, "blah is invalid type")
opts.Store.Type = "blah"
_, err = newServerApp(&opts)
assert.EqualError(t, err, "unsupported store type blah")
t.Log(err)
}
func TestServerApp_Shutdown(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts {
o.Port = 18090
return o
})
st := time.Now()
err := app.run(ctx)
assert.Nil(t, err)
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
app.Wait()
}
func TestServerApp_MainSignal(t *testing.T) {
go func() {
time.Sleep(100 * time.Millisecond)
err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
require.Nil(t, err)
}()
st := time.Now()
s := ServerOpts{}
p := flags.NewParser(&s, flags.Default)
args := []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
"--port=18100", "--url=https://demo.remark42.com"}
_, err := p.ParseArgs(args)
require.Nil(t, err)
s.Execute(args)
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
}
func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerOpts) ServerOpts) (*serverApp, context.Context) {
opts := ServerOpts{}
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
require.Nil(t, err)
opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp"
opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port)
opts.Store.Bolt.Timeout = 10 * time.Second
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid"
opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid"
opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid"
opts.BackupLocation = "/tmp"
opts = fn(opts)
os.Remove(opts.Store.Bolt.Path + "/remark.db")
// create app
app, err := newServerApp(&opts)
require.Nil(t, err)
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(duration)
log.Print("[TEST] terminate app")
cancel()
}()
return app, ctx
}
backend/app/cmd/testdata/import.txt
Vendored
+3
View File
@@ -0,0 +1,3 @@
blah
blah2
12345678
backend/app/cmd/testdata/import.txt.gz
Vendored
BIN
View File
Binary file not shown.
backend/app/main.go
+19 -431
View File
@@ -1,460 +1,48 @@
package main
import (
"context"
"fmt"
"log"
"net/url"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/coreos/bbolt"
"github.com/go-pkgz/mongo"
"github.com/hashicorp/logutils"
"github.com/jessevdk/go-flags"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest/api"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/keys"
"github.com/umputun/remark/backend/app/store/service"
"github.com/umputun/remark/backend/app/cmd"
)
// Opts with command line flags and env
// Opts has all commands
type Opts struct {
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
ServerCmd cmd.ServerOpts `command:"server"`
ImportCmd cmd.ImportCommand `command:"import"`
BackupCmd cmd.BackupCommand `command:"backup"`
RestoreCmd cmd.RestoreCommand `command:"restore"`
Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"`
Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"`
Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"`
Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"`
DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
Auth struct {
TTL struct {
JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
}
// AuthGroup defines options group for auth params
type AuthGroup struct {
CID string `long:"cid" env:"CID" description:"OAuth client ID"`
CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
}
// StoreGroup defines options group for store params
type StoreGroup struct {
Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
Bolt struct {
Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
}
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
}
// CacheGroup defines options group for cache params
type CacheGroup struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"`
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
} `group:"max" namespace:"max" env-namespace:"MAX"`
}
// MongoGroup holds all mongo params, used by store, avatar and cache
type MongoGroup struct {
URL string `long:"url" env:"URL" description:"mongo url"`
DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
}
// KeyGroup defines options group for key params
type KeyGroup struct {
Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"`
Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
}
var revision = "unknown"
// Application holds all active objects
type Application struct {
Opts
restSrv *api.Rest
migratorSrv *api.Migrator
exporter migrator.Exporter
devAuth *auth.DevAuthServer
dataService *service.DataStore
terminated chan struct{}
}
func main() {
fmt.Printf("remark %s\n", revision)
fmt.Printf("remark42 %s\n", revision)
cmd.Revision = revision
var opts Opts
p := flags.NewParser(&opts, flags.Default)
if _, e := p.ParseArgs(os.Args[1:]); e != nil {
os.Exit(1)
}
setupLog(opts.Dbg)
log.Print("[INFO] started remark")
resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
ctx, cancel := context.WithCancel(context.Background())
go func() { // catch signal and invoke graceful termination
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
<-stop
log.Print("[WARN] interrupt signal")
cancel()
}()
app, err := New(opts)
if err != nil {
log.Fatalf("[ERROR] failed to setup application, %+v", err)
}
err = app.Run(ctx)
log.Printf("[INFO] remark terminated %s", err)
}
// New prepares application and return it with all active parts
// doesn't start anything
func New(opts Opts) (*Application, error) {
if err := makeDirs(opts.BackupLocation); err != nil {
return nil, err
}
if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") {
return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
}
storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites)
if err != nil {
return nil, err
}
keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret)
if err != nil {
return nil, err
}
dataService := &service.DataStore{
Interface: storeEngine,
EditDuration: opts.EditDuration,
KeyStore: keyStore,
MaxCommentSize: opts.MaxCommentSize,
Admins: opts.Admins,
}
loadingCache, err := makeCache(opts.Cache, opts.Mongo)
if err != nil {
return nil, err
}
// token TTL is 5 minutes, inactivity interval 7+ days by default
jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie)
avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo)
if err != nil {
return nil, errors.Wrap(err, "failed to make avatar store")
}
avatarProxy := &proxy.Avatar{
Store: avatarStore,
RoutePath: "/api/v1/avatar",
RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
}
exporter := &migrator.Remark{DataStore: dataService}
migr := &api.Migrator{
Version: revision,
Cache: loadingCache,
NativeImporter: &migrator.Remark{DataStore: dataService},
DisqusImporter: &migrator.Disqus{DataStore: dataService},
WordPressImporter: &migrator.WordPress{DataStore: dataService},
NativeExported: &migrator.Remark{DataStore: dataService},
KeyStore: keyStore,
}
authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts)
imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}
commentFormatter := store.NewCommentFormatter(imgProxy)
srv := &api.Rest{
Version: revision,
DataService: dataService,
Exporter: exporter,
WebRoot: opts.WebRoot,
RemarkURL: opts.RemarkURL,
ImageProxy: imgProxy,
CommentFormatter: commentFormatter,
AvatarProxy: avatarProxy,
ReadOnlyAge: opts.ReadOnlyAge,
SharedSecret: opts.SharedSecret,
Authenticator: auth.Authenticator{
JWTService: jwtService,
AdminEmail: opts.AdminEmail,
Providers: authProviders,
DevPasswd: opts.DevPasswd,
PermissionChecker: dataService,
},
Cache: loadingCache,
}
// no admin email, use admin@domain
if srv.Authenticator.AdminEmail == "" {
if u, err := url.Parse(opts.RemarkURL); err == nil {
srv.Authenticator.AdminEmail = "admin@" + u.Host
}
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore
var devAuth *auth.DevAuthServer
if opts.Auth.Dev {
devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]}
}
tch := make(chan struct{})
return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService,
Opts: opts, terminated: tch}, nil
}
// Run all application objects
func (a *Application) Run(ctx context.Context) error {
if a.DevPasswd != "" {
log.Printf("[WARN] running in dev mode")
}
go func() {
// shutdown on context cancellation
<-ctx.Done()
a.restSrv.Shutdown()
a.migratorSrv.Shutdown()
if a.devAuth != nil {
a.devAuth.Shutdown()
}
if e := a.dataService.Close(); e != nil {
log.Printf("[WARN] failed to close store, %s", e)
}
}()
a.activateBackup(ctx) // runs in goroutine for each site
go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only
if a.Auth.Dev {
go a.devAuth.Run() // dev oauth2 server on :8084
}
a.restSrv.Run(a.Port)
close(a.terminated)
return nil
}
// Wait for application completion (termination)
func (a *Application) Wait() {
<-a.terminated
}
// activateBackup runs background backups for each site
func (a *Application) activateBackup(ctx context.Context) {
for _, siteID := range a.Sites {
backup := migrator.AutoBackup{
Exporter: a.exporter,
BackupLocation: a.BackupLocation,
SiteID: siteID,
KeepMax: a.MaxBackupFiles,
Duration: 24 * time.Hour,
}
go backup.Do(ctx)
}
}
// makeDataStore creates store for all sites
func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) {
switch group.Type {
case "bolt":
if err = makeDirs(group.Bolt.Path); err != nil {
return nil, errors.Wrap(err, "failed to create bolt store")
}
sites := []engine.BoltSite{}
for _, site := range siteNames {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
case "mongo":
mgServer, e := makeMongo(mg)
if e != nil {
return result, errors.Wrap(e, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "")
result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
default:
return nil, errors.Errorf("unsupported store type %s", group.Type)
}
return result, errors.Wrap(err, "can't initialize data store")
}
func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) {
switch group.Type {
case "fs":
if err := makeDirs(group.FS.Path); err != nil {
return nil, err
}
return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil
case "mongo":
mgServer, err := makeMongo(mg)
p.CommandHandler = func(command flags.Commander, args []string) error {
setupLog(opts.Dbg)
err := command.Execute(args)
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
log.Printf("[ERROR] failed with %+v", err)
}
conn := mongo.NewConnection(mgServer, mg.DB, "")
return avatar.NewGridFS(conn, group.RszLmt), nil
}
return nil, errors.Errorf("unsupported avatar store type %s", group.Type)
}
func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) {
switch group.Type {
case "shared":
return keys.NewStaticStore(sharedSecret), nil
default:
return nil, errors.Errorf("unsupported key store type %s", group.Type)
}
}
func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) {
switch group.Type {
case "mem":
return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
cache.MaxKeys(group.Max.Items))
case "mongo":
mgServer, err := makeMongo(mg)
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "cache")
return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
cache.MaxKeys(group.Max.Items))
}
return nil, errors.Errorf("unsupported cache type %s", group.Type)
}
// mkdir -p for all dirs
func makeDirs(dirs ...string) error {
// exists returns whether the given file or directory exists or not
exists := func(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
return err
}
for _, dir := range dirs {
ex, err := exists(dir)
if err != nil {
return errors.Wrapf(err, "can't check directory status for %s", dir)
}
if !ex {
if e := os.MkdirAll(dir, 0700); e != nil {
return errors.Wrapf(err, "can't make directory %s", dir)
}
}
}
return nil
}
func makeMongo(mg MongoGroup) (result *mongo.Server, err error) {
if mg.URL == "" {
return nil, errors.New("no mongo URL provided")
}
return mongo.NewServerWithURL(mg.URL, 10*time.Second)
}
func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
return auth.Params{
JwtService: jwtService,
AvatarProxy: avatarProxy,
RemarkURL: opts.RemarkURL,
Cid: cid,
Csecret: secret,
PermissionChecker: ds,
}
}
providers := []auth.Provider{}
if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC)))
}
if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC)))
}
if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC)))
}
if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" {
providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC)))
}
if opts.Auth.Dev {
providers = append(providers, auth.NewDev(makeParams("", "")))
}
if len(providers) == 0 {
log.Printf("[WARN] no auth providers defined")
}
return providers
}
// resetEnv clears all sensitive env vars
func resetEnv(envs ...string) {
for _, env := range envs {
if err := os.Unsetenv(env); err != nil {
log.Printf("[WARN] can't unset env %s, %s", env, err)
if _, err := p.Parse(); err != nil {
if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
os.Exit(0)
} else {
os.Exit(1)
}
}
}
backend/app/main_test.go
+24 -203
View File
@@ -1,227 +1,48 @@
package main
import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"sync"
"syscall"
"testing"
"time"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestApplication(t *testing.T) {
app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts {
o.Port = 18080
return o
})
func TestMain(t *testing.T) {
go func() { _ = app.Run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:18080/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
// add comment
resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json",
strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
require.Nil(t, err)
assert.Equal(t, http.StatusCreated, resp.StatusCode)
body, _ = ioutil.ReadAll(resp.Body)
t.Log(string(body))
assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email")
app.Wait()
}
func TestApplicationDevMode(t *testing.T) {
app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts {
o.Port = 18085
o.DevPasswd = "password"
o.Auth.Dev = true
return o
})
go func() { _ = app.Run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider")
assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider")
// send ping
resp, err := http.Get("http://localhost:18085/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
app.Wait()
}
func TestApplicationWithMongo(t *testing.T) {
mongoURL := os.Getenv("MONGO_TEST")
if mongoURL == "" {
mongoURL = "mongodb://localhost:27017/test"
}
if mongoURL == "skip" {
t.Skip("skip mongo app test")
}
opts := Opts{}
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com",
"--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"})
require.Nil(t, err)
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.BackupLocation = "/tmp"
// create app
app, err := New(opts)
require.Nil(t, err)
defer func() {
s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
assert.NoError(t, err)
conn := mongo.NewConnection(s, "test_remark", "")
_ = conn.WithDB(func(dbase *mgo.Database) error {
assert.NoError(t, dbase.DropDatabase())
return nil
})
}()
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(5 * time.Second)
log.Print("[TEST] terminate app")
cancel()
}()
go func() { _ = app.Run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:12345/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
app.Wait()
}
func TestApplicationFailed(t *testing.T) {
opts := Opts{}
p := flags.NewParser(&opts, flags.Default)
// RO bolt location
_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp",
"--store.bolt.path=/dev/null"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
"open /dev/null/remark.db: not a directory")
t.Log(err)
// RO backup location
opts = Opts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
"--backup=/dev/null/not-writable"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
t.Log(err)
// invalid url
opts = Opts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
t.Log(err)
opts = Opts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"})
assert.NotNil(t, err, "blah is invalid type")
opts.Store.Type = "blah"
_, err = New(opts)
assert.EqualError(t, err, "unsupported store type blah")
t.Log(err)
}
func TestApplicationShutdown(t *testing.T) {
app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts {
o.Port = 18090
return o
})
st := time.Now()
err := app.Run(ctx)
assert.Nil(t, err)
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
app.Wait()
}
func TestApplicationMainSignal(t *testing.T) {
os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
"--port=18100", "--url=https://demo.remark42.com"}
os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg"}
go func() {
time.Sleep(100 * time.Millisecond)
time.Sleep(500 * time.Millisecond)
err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
require.Nil(t, err)
}()
st := time.Now()
main()
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
}
func prepApp(t *testing.T, duration time.Duration, fn func(o Opts) Opts) (*Application, context.Context) {
opts := Opts{}
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
require.Nil(t, err)
opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp"
opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port)
opts.Store.Bolt.Timeout = 10 * time.Second
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid"
opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid"
opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid"
opts.BackupLocation = "/tmp"
opts = fn(opts)
os.Remove(opts.Store.Bolt.Path + "/remark.db")
// create app
app, err := New(opts)
require.Nil(t, err)
ctx, cancel := context.WithCancel(context.Background())
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
time.Sleep(duration)
log.Print("[TEST] terminate app")
cancel()
st := time.Now()
main()
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
wg.Done()
}()
return app, ctx
time.Sleep(50 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:18202/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
wg.Wait()
}
backend/app/rest/api/admin.go
+2 -2
View File
@@ -60,7 +60,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
return
}
a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last"))
a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
render.Status(r, http.StatusOK)
render.JSON(w, r, JSON{"id": id, "locator": locator})
}
@@ -129,7 +129,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
}
}
a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, "last"))
a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, lastCommentsScope))
render.Status(r, http.StatusOK)
render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID})
}
backend/app/rest/api/middleware.go
+22
View File
@@ -106,6 +106,7 @@ func Logger(ipFn func(ip string) string, flags ...LoggerFlag) func(http.Handler)
if qun, err := url.QueryUnescape(q); err == nil {
q = qun
}
q = sanitizeQuery(q)
remoteIP := strings.Split(r.RemoteAddr, ":")[0]
if strings.HasPrefix(r.RemoteAddr, "[") {
@@ -159,6 +160,27 @@ func getBodyAndUser(r *http.Request, flags []LoggerFlag) (body string, user stri
return body, user
}
func sanitizeQuery(u string) string {
out := []rune(u)
hide := []string{"password", "passwd", "secret", "credentials"}
for _, h := range hide {
if strings.Contains(strings.ToLower(u), h+"=") {
stPos := strings.Index(strings.ToLower(u), h+"=") + len(h) + 1
fnPos := strings.Index(u[stPos:], "&")
if fnPos == -1 {
fnPos = len(u)
} else {
fnPos = stPos + fnPos
}
log.Print(stPos, fnPos)
for i := stPos; i < fnPos; i++ {
out[i] = rune('*')
}
}
}
return string(out)
}
func inLogFlags(f LoggerFlag, flags []LoggerFlag) bool {
for _, flg := range flags {
if (flg == LogAll && f != LogNone) || flg == f {
backend/app/rest/api/middleware_test.go
+16
View File
@@ -66,3 +66,19 @@ func TestMiddleware_GetBodyAndUser(t *testing.T) {
assert.Equal(t, "", body)
assert.Equal(t, ` - id1 "user1"`, user, "no user")
}
func TestMiddleware_sanitizeReqURL(t *testing.T) {
tbl := []struct {
in string
out string
}{
{"", ""},
{"/aa/bb?xyz=123", "/aa/bb?xyz=123"},
{"/aa/bb?xyz=123&secret=asdfghjk", "/aa/bb?xyz=123&secret=********"},
{"/aa/bb?xyz=123&secret=asdfghjk&key=val", "/aa/bb?xyz=123&secret=********&key=val"},
{"/aa/bb?xyz=123&secret=asdfghjk&key=val&password=1234", "/aa/bb?xyz=123&secret=********&key=val&password=****"},
}
for i, tt := range tbl {
assert.Equal(t, tt.out, sanitizeQuery(tt.in), "check #%d, %s", i, tt.in)
}
}
backend/app/rest/api/migrator.go
+1 -1
View File
@@ -145,7 +145,6 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
w.WriteHeader(http.StatusOK)
gzWriter := gzip.NewWriter(w)
defer func() {
if e := gzWriter.Close(); e != nil {
@@ -154,6 +153,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
}()
if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil {
log.Printf("[WARN] can't export, %+v", err)
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
return
}
backend/app/rest/api/rest.go
+3 -1
View File
@@ -60,6 +60,8 @@ type Rest struct {
const hardBodyLimit = 1024 * 64 // limit size of body
const lastCommentsScope = "last"
type commentsWithInfo struct {
Comments []store.Comment `json:"comments"`
Info store.PostInfo `json:"info,omitempty"`
@@ -123,7 +125,7 @@ func (s *Rest) routes() chi.Router {
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"},
ExposedHeaders: []string{"Link"},
ExposedHeaders: []string{"Authorization"},
AllowCredentials: true,
MaxAge: 300,
})
backend/app/rest/api/rest_private.go
+2 -2
View File
@@ -71,7 +71,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
Scopes(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID))
Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
render.Status(r, http.StatusCreated)
render.JSON(w, r, &finalComment)
@@ -120,7 +120,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last", user.ID))
s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope, user.ID))
render.JSON(w, r, res)
}
backend/app/rest/api/rest_public.go
+2 -10
View File
@@ -114,23 +114,15 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
limit = 0
}
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes("last")
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(lastCommentsScope)
data, err := s.Cache.Get(key, func() ([]byte, error) {
comments, e := s.DataService.Last(siteID, limit)
if e != nil {
return nil, e
}
comments = s.adminService.alterComments(comments, r)
// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
filterDeleted := []store.Comment{}
for _, c := range comments {
if c.Deleted {
continue
}
filterDeleted = append(filterDeleted, c)
}
filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
return encodeJSONWithHTML(filterDeleted)
})
backend/app/rest/api/rest_public_test.go
+1
View File
@@ -238,6 +238,7 @@ func TestRest_Last(t *testing.T) {
err = json.Unmarshal([]byte(res), &comments)
assert.Nil(t, err)
assert.Equal(t, 2, len(comments), "should have 2 comments")
t.Logf("%+v", comments)
}
func TestRest_FindUserComments(t *testing.T) {
backend/app/rest/api/rest_test.go
+15
View File
@@ -72,6 +72,21 @@ func TestRest_Shutdown(t *testing.T) {
assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
}
func TestRest_filterComments(t *testing.T) {
user := store.User{ID: "user1", Name: "user name 1"}
c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
r := filterComments([]store.Comment{c1, c2, c3}, func(c store.Comment) bool {
return c.Text == "test test #1" || c.Text == "test test #3"
})
assert.Equal(t, 2, len(r), "one comment filtered")
}
func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
backend/app/rest/api/rss.go
+2 -2
View File
@@ -67,7 +67,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get rss for site %s", siteID)
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last")
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope)
data, err := s.Cache.Get(key, func() ([]byte, error) {
comments, e := s.DataService.Last(siteID, maxRssItems)
if e != nil {
@@ -100,7 +100,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last")
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope)
data, err := s.Cache.Get(key, func() (res []byte, e error) {
comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
if e != nil {
backend/app/rest/auth/auth.go
+5
View File
@@ -135,22 +135,27 @@ func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) boo
return false
}
log.Printf("[DEBUG] dev user auth")
s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
if len(s) != 2 {
log.Printf("[WARN] dev user auth failed, incorrect auth header %s", r.Header.Get("Authorization"))
return false
}
b, err := base64.StdEncoding.DecodeString(s[1])
if err != nil {
log.Printf("[WARN] dev user auth failed, failed to decode %s, %s", s[1], err)
return false
}
pair := strings.SplitN(string(b), ":", 2)
if len(pair) != 2 {
log.Printf("[WARN] dev user auth failed, failed to split %s", string(b))
return false
}
if pair[0] != "dev" || pair[1] != a.DevPasswd {
log.Printf("[WARN] dev user auth failed, user/passwd mismatch %+v", pair)
return false
}
backend/app/rest/proxy/avatar_test.go
+1 -1
View File
@@ -66,7 +66,7 @@ func TestAvatar_PutFailed(t *testing.T) {
_, err := p.Put(u)
assert.EqualError(t, err, "no picture for user1")
u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:12345/avater/pic"}
u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:22345/avater/pic"}
_, err = p.Put(u)
require.Error(t, err)
assert.Contains(t, err.Error(), "connect: connection refused")
backend/scripts/backup.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -e
/srv/remark42 backup $@
backend/scripts/create-backup.sh
-9
View File
@@ -1,9 +0,0 @@
#!/bin/sh
# this scrips makes a backup file to /srv/var/userbackup-<site>-<timestamp>.gz
set -e
BACKUP_PATH=${BACKUP_PATH:-./var}
backup_file=${BACKUP_PATH}/userbackup-${1}-$(date +%s).gz
echo "make backup file for site $1 to $backup_file"
curl "http://127.0.0.1:8081/api/v1/admin/export?site=${1}&secret=${SECRET}" > ${backup_file}
echo "created backup ${backup_file}"
backend/scripts/import-disqus.sh
-5
View File
@@ -1,5 +0,0 @@
#!/bin/sh
set -e
echo "import disqus file $1 to site $2"
curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=disqus&secret=${SECRET}"
echo "import completed"
backend/scripts/import-wordpress.sh
-5
View File
@@ -1,5 +0,0 @@
#!/bin/sh
set -e
echo "import wordpress file $1 to site $2"
curl -X POST -H "Content-Type: application/xml" --data-binary @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=wordpress&secret=${SECRET}"
echo "import completed"
backend/scripts/import.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -e
/srv/remark42 import $@
backend/scripts/migrate-data.sh
-25
View File
@@ -1,25 +0,0 @@
#!/bin/sh
# this scrips making a backup file to /tmp/export-remark.gz and loading it back
# useful to migrate data schema in case if new version of data store incomaptible with the stored comments.
set -e
echo "make backup file for site $1"
curl "http://127.0.0.1:8081/api/v1/admin/export?site=${1}&secret=${SECRET}" > /tmp/export-remark.gz
BOLTDB_PATH=${BOLTDB_PATH:-./var}
BACKUP_PATH=${BACKUP_PATH:-./var}
cp ${BOLTDB_PATH}/${1}.db ${BACKUP_PATH}/${1}-$(date +%s).db
echo "import backup to site $1"
echo "unpack /tmp/export-remark.gz"
gunzip -c /tmp/export-remark.gz >/tmp/backup.remark
ls -laH /tmp/backup.remark
echo "export to site $1"
curl -X POST -H "Content-Type: application/json" --data-binary @/tmp/backup.remark "http://127.0.0.1:8081/api/v1/admin/import?site=${1}&provider=native&secret=${SECRET}"
rm /tmp/backup.remark
rm /tmp/export-remark.gz
echo "migration completed"
backend/scripts/restore-backup.sh
-14
View File
@@ -1,14 +0,0 @@
#!/bin/sh
set -e
echo "restore backup file $1 to site $2"
BACKUP_PATH=${BACKUP_PATH:-./var}
echo "unpack $1"
gunzip -c ${BACKUP_PATH}/$1 >/tmp/backup.remark
echo "source file info"
ls -la /tmp/backup.remark
curl -X POST -H "Content-Type: application/json" --data-binary @/tmp/backup.remark "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=native&secret=${SECRET}"
rm /tmp/backup.remark
echo "backup restored"
backend/scripts/restore.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -e
/srv/remark42 restore $@
compose-dev-backend.yml
-3
View File
@@ -35,7 +35,6 @@ services:
- "8084:8084" # local oauth2 server
environment:
- USER # preset environment, UID on the host machine, i.e `id -u`
- REMARK_URL=http://127.0.0.1:8080
- SECRET=12345
- STORE_BOLT_PATH=/srv/var/db
@@ -46,5 +45,3 @@ services:
- ADMIN=dev_user # set admin flag for default user on local ouath2
volumes:
- ./var:/srv/var
command: /srv/start.sh
compose-dev-frontend.yml
-3
View File
@@ -28,7 +28,6 @@ services:
- "8084:8084" # local oauth2 server
environment:
- USER # preset environment, UID on the host machine, i.e `id -u`
- REMARK_URL=http://127.0.0.1:8080
- SECRET=12345
- STORE_BOLT_PATH=/srv/var/db
@@ -39,5 +38,3 @@ services:
- ADMIN=dev_user # set admin flag for default user on local ouath2
volumes:
- ./var:/srv/var
command: /srv/start.sh
docker-compose.yml
-3
View File
@@ -19,7 +19,6 @@ services:
# - "80:8080"
environment:
- USER
- REMARK_URL
- SECRET
- STORE_BOLT_PATH=/srv/var/db
@@ -36,5 +35,3 @@ services:
# - DEV_PASSWD=password # development mode, be careful!
volumes:
- ./var:/srv/var
command: /srv/start.sh
start.sh → entrypoint.sh
+3 -11
View File
@@ -1,19 +1,11 @@
#!/bin/sh
echo "prepare environment"
# replace BASE_URL constant by REMARK_URL
sed -i "s|https://demo.remark42.com|${REMARK_URL}|g" /srv/web/*.js
# remove devtools attach helper. TODO: move to webpack loader
sed -i "/REMOVE-START/,/REMOVE-END/d" /srv/web/iframe.html
chown -R app:app /srv/var 2>/dev/null
echo "start remark42 server"
if [ -z "$USER" ] ; then \
echo "No USER defined, runs under root!"
exec /srv/remark
else
echo "runs under ${USER}"
/sbin/su-exec ${USER} /srv/remark
fi
/sbin/su-exec app /srv/remark42 $@