From b076a06e33254141d4f14f69ab0c3d2b4beacd18 Mon Sep 17 00:00:00 2001 From: Aleksey Gurianov Date: Tue, 10 Jul 2018 14:13:53 +0400 Subject: [PATCH 01/93] regenerate package-lock --- web/package-lock.json | 22 ++++++++ web/yarn-error.log | 128 ------------------------------------------ 2 files changed, 22 insertions(+), 128 deletions(-) delete mode 100644 web/yarn-error.log diff --git a/web/package-lock.json b/web/package-lock.json index e76274a2..af09fa64 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -6634,6 +6634,12 @@ } } }, + "jest-localstorage-mock": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/jest-localstorage-mock/-/jest-localstorage-mock-2.2.0.tgz", + "integrity": "sha512-x+P0vcwr4540bCAYzTEpiD9rs+zh/QZzyiABV+MU6yM2OPwPlrrLyUx/6gValMyt6tg5lX6Z53o2rHWfUht5Xw==", + "dev": true + }, "jest-matcher-utils": { "version": "23.0.1", "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-23.0.1.tgz", @@ -11555,6 +11561,12 @@ "resolved": "https://registry.npmjs.org/preact/-/preact-8.2.9.tgz", "integrity": "sha512-ThuGXBmJS3VsT+jIP+eQufD3L8pRw/PY3FoCys6O9Pu6aF12Pn9zAJDX99TfwRAFOCEKm/P0lwiPTbqKMJp0fA==" }, + "preact-redux": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/preact-redux/-/preact-redux-2.0.3.tgz", + "integrity": "sha1-lgpTXDImQ801mY8z8MLme8Hn6qs=", + "dev": true + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -11927,6 +11939,16 @@ } } }, + "redux": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/redux/-/redux-4.0.0.tgz", + "integrity": "sha512-NnnHF0h0WVE/hXyrB6OlX67LYRuaf/rJcbWvnHHEPCF/Xa/AZpwhs/20WyqzQae5x4SD2F9nPObgBh2rxAgLiA==", + "dev": true, + "requires": { + "loose-envify": "^1.1.0", + "symbol-observable": "^1.2.0" + } + }, "regenerate": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.3.3.tgz", diff --git a/web/yarn-error.log b/web/yarn-error.log deleted file mode 100644 index 2e432a7f..00000000 --- a/web/yarn-error.log +++ /dev/null @@ -1,128 +0,0 @@ -Arguments: - /Users/dmitrytsepelev/.nvm/versions/node/v9.2.0/bin/node /Users/dmitrytsepelev/.nvm/versions/node/v9.2.0/bin/yarn start - -PATH: - /anaconda2/bin:/Users/dmitrytsepelev/.rvm/gems/ruby-2.5.1/bin:/Users/dmitrytsepelev/.rvm/gems/ruby-2.5.1@global/bin:/Users/dmitrytsepelev/.rvm/rubies/ruby-2.5.1/bin:/usr/local/heroku/bin:/Users/dmitrytsepelev/.nvm/versions/node/v9.2.0/bin:./bin:bin:/Users/dmitrytsepelev/bin:/usr/local/bin:/usr/local/sbin:/usr/local/share/npm/bin:/usr/bin:/bin:/usr/sbin:/sbin::/Users/dmitrytsepelev/.rvm/bin:/usr/local/go/bin:/Users/dmitrytsepelev/go/bin:/Users/dmitrytsepelev/Library/Android/sdk/tools:/Users/dmitrytsepelev/Library/Android/sdk/platform-tools:/Users/dmitrytsepelev/Library/Android/sdk/tools/proguard/bin - -Yarn version: - 1.5.1 - -Node version: - 9.2.0 - -Platform: - darwin x64 - -npm manifest: - { - "name": "remark-ui", - "version": "0.1.0", - "scripts": { - "build": "cross-env NODE_ENV=production webpack --config ./webpack.config.js", - "start": "webpack-dev-server --progress --hot --inline --config ./webpack.config.js", - "lint": "eslint --ext=.js,.jsx .", - "test": "jest", - "prettier": "prettier --write \"./**/*.{js,jsx,scss}\"", - "precommit": "./node_modules/.bin/lint-staged" - }, - "lint-staged": { - "./**/*.{js,jsx}": [ - "eslint --fix", - "git add" - ], - "./**/*.scss": [ - "prettier --write", - "git add" - ] - }, - "devDependencies": { - "autoprefixer": "^7.2.6", - "babel-core": "^6.26.3", - "babel-eslint": "^8.2.5", - "babel-loader": "^7.1.4", - "babel-plugin-syntax-dynamic-import": "^6.18.0", - "babel-plugin-transform-object-rest-spread": "^6.26.0", - "babel-plugin-transform-react-jsx": "^6.24.1", - "babel-preset-env": "^1.7.0", - "clean-webpack-plugin": "^0.1.19", - "copy-webpack-plugin": "^4.5.1", - "core-js": "^2.5.7", - "cross-env": "^5.2.0", - "css-loader": "^0.28.11", - "eslint": "^4.19.1", - "eslint-config-prettier": "^2.9.0", - "eslint-plugin-jsx-a11y": "^6.1.0", - "eslint-plugin-prettier": "^2.6.1", - "eslint-plugin-react": "^7.10.0", - "extract-text-webpack-plugin": "^3.0.2", - "file-loader": "^0.11.1", - "html-webpack-plugin": "^2.30.1", - "husky": "^0.14.3", - "jest": "^23.1.0", - "jest-localstorage-mock": "^2.2.0", - "lint-staged": "^7.2.0", - "postcss-calc": "^6.0.1", - "postcss-csso": "^2.0.0", - "postcss-for": "^2.1.1", - "postcss-loader": "^2.1.5", - "postcss-nested": "^3.0.0", - "postcss-simple-vars": "^4.1.0", - "postcss-url": "^6.3.1", - "postcss-wrap": "0.0.4", - "preact-redux": "^2.0.3", - "prettier": "^1.13.7", - "redux": "^4.0.0", - "style-loader": "^0.19.1", - "webpack": "^3.12.0", - "webpack-bundle-analyzer": "^2.13.1", - "webpack-dev-server": "^2.7.1" - }, - "dependencies": { - "axios": "^0.18.0", - "bem-react-helper": "^1.1.2", - "preact": "^8.2.9" - }, - "eslintIgnore": [ - "public" - ], - "jest": { - "transform": { - "^.+\\.jsx?$": "/fileTransformer.js" - }, - "setupFiles": [ - "/injectGlobalVariable.js", - "jest-localstorage-mock" - ], - "moduleDirectories": [ - "node_modules", - "/app" - ], - "testMatch": [ - "/**/*.test.js" - ] - }, - "engines": { - "node": ">=8" - } - } - -yarn manifest: - No manifest - -Lockfile: - No lockfile - -Trace: - Error: Command failed. - Exit code: 1 - Command: sh - Arguments: -c webpack-dev-server --progress --hot --inline --config ./webpack.config.js - Directory: /Users/dmitrytsepelev/dev/remark/web - Output: - - at ProcessTermError.MessageError (/Users/dmitrytsepelev/.nvm/versions/node/v9.2.0/lib/node_modules/yarn/lib/cli.js:186:110) - at new ProcessTermError (/Users/dmitrytsepelev/.nvm/versions/node/v9.2.0/lib/node_modules/yarn/lib/cli.js:226:113) - at ChildProcess. (/Users/dmitrytsepelev/.nvm/versions/node/v9.2.0/lib/node_modules/yarn/lib/cli.js:30281:17) - at ChildProcess.emit (events.js:159:13) - at maybeClose (internal/child_process.js:943:16) - at Process.ChildProcess._handle.onexit (internal/child_process.js:220:5) From b2a8434f9a004184e86b15d3491dc23e75a61f96 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 10 Jul 2018 12:19:10 -0500 Subject: [PATCH 02/93] add automatic github-style avatar for dev auth #106 --- backend/Gopkg.lock | 8 +- backend/app/rest/auth/dev_provider.go | 62 ++- backend/app/rest/auth/dev_provider_test.go | 10 +- .../github.com/nullrocks/identicon/.gitignore | 33 ++ .../github.com/nullrocks/identicon/LICENSE | 21 + .../github.com/nullrocks/identicon/README.md | 91 +++++ .../nullrocks/identicon/_config.yml | 1 + .../github.com/nullrocks/identicon/canvas.go | 121 ++++++ .../github.com/nullrocks/identicon/config.yml | 11 + .../nullrocks/identicon/generator.go | 170 ++++++++ .../github.com/nullrocks/identicon/hsl.go | 83 ++++ .../nullrocks/identicon/identicon.go | 378 ++++++++++++++++++ .../github.com/nullrocks/identicon/image.go | 73 ++++ .../github.com/nullrocks/identicon/svg.go | 92 +++++ 14 files changed, 1140 insertions(+), 14 deletions(-) create mode 100644 backend/vendor/github.com/nullrocks/identicon/.gitignore create mode 100644 backend/vendor/github.com/nullrocks/identicon/LICENSE create mode 100644 backend/vendor/github.com/nullrocks/identicon/README.md create mode 100644 backend/vendor/github.com/nullrocks/identicon/_config.yml create mode 100644 backend/vendor/github.com/nullrocks/identicon/canvas.go create mode 100644 backend/vendor/github.com/nullrocks/identicon/config.yml create mode 100644 backend/vendor/github.com/nullrocks/identicon/generator.go create mode 100644 backend/vendor/github.com/nullrocks/identicon/hsl.go create mode 100644 backend/vendor/github.com/nullrocks/identicon/identicon.go create mode 100644 backend/vendor/github.com/nullrocks/identicon/image.go create mode 100644 backend/vendor/github.com/nullrocks/identicon/svg.go diff --git a/backend/Gopkg.lock b/backend/Gopkg.lock index 586a3e64..41b0685d 100644 --- a/backend/Gopkg.lock +++ b/backend/Gopkg.lock @@ -126,6 +126,12 @@ packages = ["."] revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b" +[[projects]] + branch = "master" + name = "github.com/nullrocks/identicon" + packages = ["."] + revision = "7875f45b0022edded6377e40639d8aa620193a62" + [[projects]] name = "github.com/patrickmn/go-cache" packages = ["."] @@ -232,6 +238,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "af8b7f1817ce6e82746722a745184bd50c341733bcc024311aa956ea32475796" + inputs-digest = "38280a8f373ce1799e64ac1709e9748ae8cf6cca30dfeabca807d2db9fe34ab8" solver-name = "gps-cdcl" solver-version = 1 diff --git a/backend/app/rest/auth/dev_provider.go b/backend/app/rest/auth/dev_provider.go index 985f0530..0d5b111f 100644 --- a/backend/app/rest/auth/dev_provider.go +++ b/backend/app/rest/auth/dev_provider.go @@ -1,6 +1,7 @@ package auth import ( + "bytes" "context" "fmt" "log" @@ -9,6 +10,8 @@ import ( "sync" "time" + "github.com/nullrocks/identicon" + "github.com/pkg/errors" "golang.org/x/oauth2" "github.com/umputun/remark/backend/app/store" @@ -19,22 +22,28 @@ const devAuthPort = 8084 // DevAuthServer is a fake oauth server for development // it provides stand-alone server running on its own port and pretending to be the real oauth2. It also provides // Dev Provider the same way as normal providers di, i.e. github, google and others. -// can run in interractive and non-interactive mode. In interactive mode login attempts will show login form to select +// can run in interactive and non-interactive mode. In interactive mode login attempts will show login form to select // desired user name. type DevAuthServer struct { Provider Provider username string // unsafe, but fine for dev nonInteractive bool - - httpServer *http.Server - lock sync.Mutex + iconGen *identicon.Generator + httpServer *http.Server + lock sync.Mutex } // Run oauth2 dev server on port devAuthPort func (d *DevAuthServer) Run() { log.Printf("[INFO] run local oauth2 dev server on %d", devAuthPort) d.lock.Lock() + var err error + d.iconGen, err = identicon.New("github", 5, 3) + if err != nil { + log.Printf("[WARN] can't create identicon, %s", err) + } + d.httpServer = &http.Server{ Addr: fmt.Sprintf(":%d", devAuthPort), Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -42,9 +51,9 @@ func (d *DevAuthServer) Run() { switch { case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"): - // first time it will be called without usernam and will ask for onw + // first time it will be called without username and will ask for onw if !d.nonInteractive && (r.ParseForm() != nil || r.Form.Get("username") == "") { - if _, err := w.Write([]byte(fmt.Sprintf(devUserForm, r.URL.RawQuery))); err != nil { + if _, err = w.Write([]byte(fmt.Sprintf(devUserForm, r.URL.RawQuery))); err != nil { log.Printf("[WARN] can't write, %s", err) } return @@ -70,19 +79,33 @@ func (d *DevAuthServer) Run() { "state":"12345678" }` w.Header().Set("Content-Type", "application/json; charset=utf-8") - if _, err := w.Write([]byte(res)); err != nil { + if _, err = w.Write([]byte(res)); err != nil { w.WriteHeader(http.StatusInternalServerError) return } case strings.HasPrefix(r.URL.Path, "/user"): + ava := fmt.Sprintf("http://127.0.0.1:%d/avatar?user=%s", devAuthPort, d.username) res := fmt.Sprintf(`{ "id": "%s", - "name":"%s" - }`, d.username, d.username) + "name":"%s", + "picture":"%s" + }`, d.username, d.username, ava) w.Header().Set("Content-Type", "application/json; charset=utf-8") - if _, err := w.Write([]byte(res)); err != nil { + if _, err = w.Write([]byte(res)); err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + + case strings.HasPrefix(r.URL.Path, "/avatar"): + user := r.URL.Query().Get("user") + b, e := d.genAvatar(user) + if e != nil { + w.WriteHeader(http.StatusNotFound) + return + } + if _, err = w.Write(b); err != nil { w.WriteHeader(http.StatusInternalServerError) return } @@ -94,7 +117,7 @@ func (d *DevAuthServer) Run() { } d.lock.Unlock() - err := d.httpServer.ListenAndServe() + err = d.httpServer.ListenAndServe() log.Printf("[WARN] dev oauth2 server terminated, %s", err) } @@ -128,13 +151,28 @@ func NewDev(p Params) Provider { userInfo := store.User{ ID: data.value("id"), Name: data.value("name"), - Picture: "", + Picture: data.value("picture"), } return userInfo }, }) } +func (d *DevAuthServer) genAvatar(user string) ([]byte, error) { + if d.iconGen == nil { + return nil, errors.Errorf("no iconGen, skip avatar generation for %s", user) + } + + ii, err := d.iconGen.Draw(user) // Generate an IdentIcon + if err != nil { + return nil, errors.Wrapf(err, "failed to draqw avatar for %s", user) + } + + buf := &bytes.Buffer{} + err = ii.Png(300, buf) + return buf.Bytes(), err +} + var devUserForm = ` diff --git a/backend/app/rest/auth/dev_provider_test.go b/backend/app/rest/auth/dev_provider_test.go index 831ff3c1..448ac29c 100644 --- a/backend/app/rest/auth/dev_provider_test.go +++ b/backend/app/rest/auth/dev_provider_test.go @@ -62,7 +62,15 @@ func TestDevProvider(t *testing.T) { assert.Nil(t, err) u := *claims.User - assert.Equal(t, store.User{Name: "dev_user", ID: "dev_user", Picture: "", IP: "", + assert.Equal(t, store.User{Name: "dev_user", ID: "dev_user", Picture: "http://127.0.0.1:8084/avatar?user=dev_user", IP: "", Admin: true, Blocked: false, Verified: false}, u) + // check avatar + resp, err = client.Get("http://127.0.0.1:8084/avatar?user=dev_user") + require.Nil(t, err) + assert.Equal(t, 200, resp.StatusCode) + body, err = ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, 985, len(body)) + t.Logf("headers: %+v", resp.Header) } diff --git a/backend/vendor/github.com/nullrocks/identicon/.gitignore b/backend/vendor/github.com/nullrocks/identicon/.gitignore new file mode 100644 index 00000000..8344b618 --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/.gitignore @@ -0,0 +1,33 @@ +# Testing and Developing +main/ +experimental/ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +.vscode +.idea + +# IDEs and editors +/.idea +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace +.vscode/* + +# System Files +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/backend/vendor/github.com/nullrocks/identicon/LICENSE b/backend/vendor/github.com/nullrocks/identicon/LICENSE new file mode 100644 index 00000000..ab277bad --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Ruben Rivera + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/backend/vendor/github.com/nullrocks/identicon/README.md b/backend/vendor/github.com/nullrocks/identicon/README.md new file mode 100644 index 00000000..67ef8c4b --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/README.md @@ -0,0 +1,91 @@ +# IdentIcon + + +![CircleCI](https://img.shields.io/circleci/project/github/RedSparr0w/node-csgo-parser.svg) + [![Go Report Card](https://goreportcard.com/badge/github.com/nullrocks/identicon)](https://goreportcard.com/report/github.com/nullrocks/identicon) [![](https://godoc.org/github.com/nullrocks/identicon?status.svg)](http://godoc.org/github.com/nullrocks/identicon) + +**IdentIcon** is an open source avatar generator inspired by GitHub avatars. + + +IdentIcon uses a deterministic algorithm that generates an image (using Golang's stdlib image encoders) based on a text (Generally Usernames, Emails or just random strings), by hashing it and iterating over the bytes of the digest to pick whether to draw a point, pick a color or choose where to go next. + + +IdentIcon's Generator enables the creation of customized figures: (NxN size, points density, custom color palette) as well as multiple exporting formats in case the developers want to generate their own images. + +## Installation +```bash +$ go get github.com/nullrocks/identicon +``` + +## Usage + +```go + +import ( + "os" + + "github.com/nullrocks/identicon" +) + +// New Generator: Rehuse +ig, err := identicon.New( + "github", // Namespace + 5, // Number of blocks (Size) + 3, // Density +) + +if err != nil { + panic(err) // Invalid Size or Density +} + +username := "nullrocks" // Text - decides the resulting figure +ii, err := ig.Draw(username) // Generate an IdentIcon + +if err != nil { + panic(err) // Text is empty +} + +// File writer +img, _ := os.Create("icon.png") +defer img.Close() +// Takes the size in pixels and any io.Writer +ii.Png(300, img) // 300px * 300px + +``` + +## Examples + +### 5x5 +|nullrocks | johndoe | abc123 | modulo | +:------------------------------------------------:|:---------------------------------------------:|:-------------------------------------------:|:--------------------------------------------| +![nullrocks](./examples/5x5/nullrocks.png) | ![johndoe](./examples/5x5/johndoe.png) | ![abc123](./examples/5x5/abc123.png) | ![modulo](./examples/5x5/modulo.png) | +![nullrocks](./examples/5x5/nullrocks_itx.png) | ![johndoe](./examples/5x5/johndoe_itx.png) | ![abc123](./examples/5x5/abc123_itx.png) | ![modulo](./examples/5x5/modulo_itx.png) | +![nullrocks](./examples/5x5/nullrocks_github.png) | ![johndoe](./examples/5x5/johndoe_github.png) | ![abc123](./examples/5x5/abc123_github.png) | ![modulo](./examples/5x5/modulo_github.png) | + +### 7x7 +|nullrocks | johndoe | abc123 | modulo | +:------------------------------------------------:|:---------------------------------------------:|:-------------------------------------------:|:---------------------------------------------| +![nullrocks](./examples/7x7/nullrocks.png) | ![johndoe](./examples/7x7/johndoe.png) | ![abc123](./examples/7x7/abc123.png) | ![modulo](./examples/7x7/modulo.png) | +![nullrocks](./examples/7x7/nullrocks_itx.png) | ![johndoe](./examples/7x7/johndoe_itx.png) | ![abc123](./examples/7x7/abc123_itx.png) | ![modulo](./examples/7x7/modulo_itx.png) | +![nullrocks](./examples/7x7/nullrocks_github.png) | ![johndoe](./examples/7x7/johndoe_github.png) | ![abc123](./examples/7x7/abc123_github.png) | ![modulo](./examples/7x7/modulo_github.png) | + +### 10x10 +|nullrocks | johndoe | abc123 | modulo | +:--------------------------------------------------:|:-----------------------------------------------:|:---------------------------------------------:|:----------------------------------------------| +![nullrocks](./examples/10x10/nullrocks.png) | ![johndoe](./examples/10x10/johndoe.png) | ![abc123](./examples/10x10/abc123.png) | ![modulo](./examples/10x10/modulo.png) | +![nullrocks](./examples/10x10/nullrocks_itx.png) | ![johndoe](./examples/10x10/johndoe_itx.png) | ![abc123](./examples/10x10/abc123_itx.png) | ![modulo](./examples/10x10/modulo_itx.png) | +![nullrocks](./examples/10x10/nullrocks_github.png) | ![johndoe](./examples/10x10/johndoe_github.png) | ![abc123](./examples/10x10/abc123_github.png) | ![modulo](./examples/10x10/modulo_github.png) | + +[View examples](./examples) + +## Documentation + +## Changelog + +## Contribution + +## License +MIT + +Copyright (c) 2018-present, Ruben Rivera + diff --git a/backend/vendor/github.com/nullrocks/identicon/_config.yml b/backend/vendor/github.com/nullrocks/identicon/_config.yml new file mode 100644 index 00000000..c4192631 --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-cayman \ No newline at end of file diff --git a/backend/vendor/github.com/nullrocks/identicon/canvas.go b/backend/vendor/github.com/nullrocks/identicon/canvas.go new file mode 100644 index 00000000..eddd3faa --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/canvas.go @@ -0,0 +1,121 @@ +package identicon + +import ( + "image" + "strconv" +) + +// Canvas contains what is needed to generate an image. It contains properties +// that could be useful when rendering the image. +// - Having MinY and MaxY allows you to vertically center the figure. +// - VisitedYPoints could be useful to determine whether there is a big empty +// vertical space in the figure. +type Canvas struct { + // Size same value specified in identicon.New(...). + Size int + // PointsMap contains all coordinates and it's values that form the figure. + PointsMap map[int]map[int]int + // MinY is the upper Y-axis that has at least one point drawn. + MinY int + // MaxY is the lower Y-axis that has at least one point drawn. + MaxY int + // VisitedYPoints contains all Y-axis that had been visited. Helpful to + // determine big blank spaces in the resulting figure. + VisitedYPoints map[int]bool + // FilledPoints is the number of points filled at least once. + FilledPoints int +} + +// Array generates a two-dimensional array version of the IdentIcon figure. +func (c *Canvas) Array() [][]int { + canvasArray := make([][]int, c.Size) + for i := range canvasArray { + canvasArray[i] = make([]int, c.Size) + } + + for y := range c.PointsMap { + for x := range c.PointsMap[y] { + canvasArray[y][x] = c.PointsMap[y][x] + } + } + + return canvasArray +} + +// ToString generates a string version of the IdentIcon figure. +func (c *Canvas) String(separator string, fillEmptyWith string) string { + tp := c.Size * c.Size + // Total number of characters considering: + strLen := c.Size - 1 // Line Breaks + strLen += (tp - c.Size) * len(separator) // Separators + strLen += c.FilledPoints // Points + strLen += (tp - c.FilledPoints) * len(fillEmptyWith) // Fill Empty + + // Concatenating strings with the `+` is slow and uses a lot of memory, + // using `copy` in a slice of bytes has been proved to be a better approach. + bs := make([]byte, strLen) + // Keep track of the length of the bytes array, concatenations will occur + // using `bl` as the right-most index. + bl := 0 + + for y := 0; y < c.Size; y++ { + if mapY, exists := c.PointsMap[y]; exists { + for x := 0; x < c.Size; x++ { + if value, exists := mapY[x]; exists { + if value > 9 { + value = 9 + } + bl += copy(bs[bl:], []byte(strconv.Itoa(value))) + } else { + bl += copy(bs[bl:], []byte(fillEmptyWith)) + } + if x < c.Size-1 { + bl += copy(bs[bl:], []byte(separator)) + } + } + } else { + // There aren't any values in this row, fill it the row anyway. + for x := 0; x < c.Size; x++ { + bl += copy(bs[bl:], []byte(fillEmptyWith)) + if x < c.Size-1 { + bl += copy(bs[bl:], []byte(separator)) + } + } + } + + if y < c.Size-1 { + // Append a line break except when it's the last line. + bl += copy(bs[bl:], "\n") + } + } + + return string(bs) +} + +// Points generates an array of points of a two-dimensional plane as [x, y] +// that correspond to all filled points in the IdentIcon figure. +func (c *Canvas) Points() []image.Point { + points := []image.Point{} + + for y, value := range c.PointsMap { + for x := range value { + points = append(points, image.Point{X: x, Y: y}) + } + } + + return points +} + +// IntCoordinates generates an array of points of a two-dimensional plane as: +// - [x, y] that correspond to all filled points in the IdentIcon figure. +func (c *Canvas) IntCoordinates() [][]int { + points := [][]int{} + + for y, value := range c.PointsMap { + for x := range value { + points = append(points, []int{x, y}) + } + } + + return points +} diff --git a/backend/vendor/github.com/nullrocks/identicon/config.yml b/backend/vendor/github.com/nullrocks/identicon/config.yml new file mode 100644 index 00000000..e79a1b85 --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/config.yml @@ -0,0 +1,11 @@ +# Golang CircleCI 2.0 configuration file +version: 2 +jobs: + build: + docker: + - image: circleci/golang:1.10 + working_directory: /go/src/github.com/nullrocks/identicon + steps: + - checkout + - run: go get -v -t -d ./... + - run: go test -v ./... \ No newline at end of file diff --git a/backend/vendor/github.com/nullrocks/identicon/generator.go b/backend/vendor/github.com/nullrocks/identicon/generator.go new file mode 100644 index 00000000..7b024a2b --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/generator.go @@ -0,0 +1,170 @@ +package identicon + +import ( + "crypto/sha256" + "errors" + "image/color" + "math/rand" + "strconv" + "time" +) + +// Generator represents a predefined set of configurations that can be reused to +// create multiple icons by passing a Text string only. +type Generator struct { + // Namespace that will be concatenated previous to the icon generation. + Namespace string + // Size is the number of blocks of the figure. + Size int + // Density * Size = times to iterate over the hash of Text:Namespace:Seed. + Density int + // hashFunction used to generate a fixed length array of bytes. + hashFunction func([]byte) []byte + // fillColorFunction used to pick a color to fill the squares of the figure. + fillColorFunction func([]byte) color.Color + // backgroundColorFunction used to pick a background color for the figure. + backgroundColorFunction func([]byte, color.Color) color.Color + // isRandom flag to decide whether the generated image will be randomized. + isRandom bool + // rand is the source of randomness. + rand *rand.Rand +} + +// option Configuration functional approach +type option func(*Generator) + +// New returns a pointer to a Generator with the desired configuration. +func New( + namespace string, + size int, + density int, + opts ...option, +) (*Generator, error) { + + if size < MinSize { + // Smaller values will generate a meaningless Generator. + return nil, errors.New( + "Size cannot be less than " + strconv.Itoa(MinSize), + ) + } + + if density < 1 { + return nil, errors.New( + "Density cannot be less than 1", + ) + } + + g := Generator{ + Size: size, + Namespace: namespace, + Density: density, + isRandom: false, + hashFunction: _sha256, + fillColorFunction: _fillColor, + backgroundColorFunction: _backgroundColor, + } + + g.Option(opts...) + + return &g, nil +} + +// Draw returns a pointer to an IdentIcon with a generated figure and a color. +func (g Generator) Draw(text string) (*IdentIcon, error) { + + var randomGenerator *rand.Rand + + if g.isRandom { + // In order to generate a randomized Canvas, use UnixNano as the source + // of randomess. By doing this, random numbers won't be consistent. + randomGenerator = rand.New(rand.NewSource(time.Now().UnixNano())) + } else { + // rand will generate consistent values since the source is Size. + randomGenerator = rand.New(rand.NewSource(int64(g.Size))) + } + + ii, err := newIdentIcon( + text, + g.Namespace, + g.Size, + g.Density, + g.isRandom, + randomGenerator, + g.hashFunction, + g.fillColorFunction, + g.backgroundColorFunction, + ) + + if err != nil { + return nil, err + } + + // Generate Canvas + ii.Draw() + + return ii, nil +} + +// Option sets the options specified. +func (g *Generator) Option(opts ...option) { + for _, opt := range opts { + opt(g) + } +} + +// SetHashFunction replaces the default hash function (Sha256). +func SetHashFunction(hf func([]byte) []byte) option { + return func(g *Generator) { + g.hashFunction = hf + } +} + +// SetFillColorFunction replaces the default color generation function (HSL). +func SetFillColorFunction(fcf func([]byte) color.Color) option { + return func(g *Generator) { + g.fillColorFunction = fcf + } +} + +// SetBackgroundColorFunction replaces the default background's color generation +// function (HSL). +func SetBackgroundColorFunction(bcf func([]byte, color.Color) color.Color) option { + return func(g *Generator) { + g.backgroundColorFunction = bcf + } +} + +// SetRandom to append a random string to the generator text everytime Draw is +// called. +func SetRandom(r bool) option { + return func(g *Generator) { + g.isRandom = r + } +} + +func _sha256(b []byte) []byte { + digest := sha256.Sum256(b) + return digest[:] +} + +func _fillColor(hashBytes []byte) color.Color { + cb1, cb2 := uint32(hashBytes[0]), uint32(hashBytes[1]) + h := (cb1 + cb2) % 360 + s := (cb1 % 30) + 60 + l := (cb2 % 20) + 40 + + // Some colors in the HSL color model are too bright and don't play well + // with the default background color. This is a naïve normalization method. + if (h >= 50 && h <= 85) || (h >= 170 && h <= 190) { + s = 80 + l -= 20 + } else if h > 85 && h < 170 { + l -= 10 + } + + return HSL{h, s, l} +} + +func _backgroundColor(hashBytes []byte, fill color.Color) color.Color { + return color.NRGBA{R: 240, G: 240, B: 240, A: 255} +} diff --git a/backend/vendor/github.com/nullrocks/identicon/hsl.go b/backend/vendor/github.com/nullrocks/identicon/hsl.go new file mode 100644 index 00000000..b01d95c2 --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/hsl.go @@ -0,0 +1,83 @@ +package identicon + +// Identicon WebColor maxium values. +const ( + // hueMax is the maximum allowed value for Hue in the HSL color model. + hueMax = 360 + // saturationMax is the maximum allowed value for Saturation in the HSL + // color model. + saturationMax = 100 + // lightnessMax is the maximum allowed value for lightnessMax in the HSL + // color model. + lightnessMax = 100 + // rgbaMax is the maximum allowed value for any R, G, B, A property value. + rgbaMax = 255 +) + +// HSL is a color model representation based on RGB. HSL facilitates the +// generation of colors that look similar between themselves by changing the +// value of Hue H while keeping Saturation S and Lightness L the same. +type HSL struct { + // Hue [0, 360] + H uint32 + // Saturation [0, 100] + S uint32 + // Lightness [0, 100] + L uint32 +} + +// RGBA conversion +func (hsl HSL) RGBA() (r, g, b, a uint32) { + h := 1.0 / float64(hueMax) * float64(hsl.H) + s := float64(hsl.S) / float64(saturationMax) + l := float64(hsl.L) / float64(lightnessMax) + r, g, b = hslToRgb(h, s, l) + a = rgbaMax + r |= r << 8 + g |= g << 8 + b |= b << 8 + a |= a << 8 + return +} + +// Golang port of Mohen's code in Stack Overflow. +// https://stackoverflow.com/questions/2353211/hsl-to-rgb-color-conversion +func hslToRgb(h, s, l float64) (uint32, uint32, uint32) { + var q, p float64 + var r, g, b float64 + + if s == 0 { + r = l + g = l + b = l + } else { + if l < 0.5 { + q = l * (1 + s) + } else { + q = (l + s) - (l * s) + } + p = (2 * l) - q + r = hueToRgb(p, q, h+(1.0/3.0)) + g = hueToRgb(p, q, h) + b = hueToRgb(p, q, h-(1.0/3.0)) + } + + return uint32(r * rgbaMax), uint32(g * rgbaMax), uint32(b * rgbaMax) +} + +func hueToRgb(p, q, t float64) float64 { + if t < 0 { + t++ + } else if t > 1 { + t-- + } + switch { + case 6*t < 1: + return (p + (q-p)*6*t) + case 2*t < 1: + return q + case 3*t < 2: + return p + (q-p)*((2.0/3.0)-t)*6 + } + return p +} diff --git a/backend/vendor/github.com/nullrocks/identicon/identicon.go b/backend/vendor/github.com/nullrocks/identicon/identicon.go new file mode 100644 index 00000000..adcb118f --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/identicon.go @@ -0,0 +1,378 @@ +// Package identicon is an open source avatar generator inspired by GitHub avatars. +// +// IdentIcon uses a deterministic algorithm that generates an image (using Golang's +// stdlib image encoders) based on a text (Generally Usernames, Emails or just +// random strings), by hashing it and iterating over the bytes of the digest to pick +// whether to draw a point, pick a color or choose where to go next. +// +// IdentIcon's Generator enables the creation of customized figures: (NxN size, +// points density, custom color palette) as well as multiple exporting formats in +// case the developers want to generate their own images. +package identicon + +import ( + "errors" + "image" + "image/color" + "math/rand" + "strconv" +) + +const ( + // Bits used to give continuity + moveUp = 0x80 + moveDown = 0x40 + moveLeft = 0x20 + moveRight = 0x10 + + // Either 0x8 or 0x2 are active + fillPoint = 0xA +) + +// Constrains for the size of the IdentIcon. +const ( + // MinSize is the minimal number of blocks allowed, anything lower that this + // wouldn't make sense. + MinSize = 4 +) + +// IdentIcon represents a mirror-symmetry image generated from a string and a +// set of configurations. +type IdentIcon struct { + // Text is the base string that will generate the canvas after being hashed. + Text string + // Namespace + Namespace string + + // Size is the number of blocks of the figure. + Size int + // Density * Size = times to iterate over the hash of Text. + Density int + // Canvas is a map of maps that contains the points and values that has been + // visited and filled. + Canvas Canvas + + // FillColor is the color used to fill squares in the figure when encoding + // to PNG or JPEG. + FillColor color.Color + // BackgroundColor is the background color of the figure when encoding it to + // PNG or JPEG. + BackgroundColor color.Color + // fillColorFunction used to pick a color to fill the squares of the figure. + fillColorFunction func([]byte) color.Color + // backgroundColorFunction used to pick a background color for the figure. + backgroundColorFunction func([]byte, color.Color) color.Color + + // drawableWidth represents the length of the left half of the canvas. + drawableWidth int + // hasBeenDrawn indicates whether the Draw() has been called before. + hasBeenDrawn bool + + // hashFunction used to generate a fixed length array of bytes. + hashFunction func([]byte) []byte + + // isRandom flag to decide whether the generated image will be randomized. + isRandom bool + // randomSeed + randomSeed string + // rand is the source of randomness. + rand *rand.Rand +} + +// Draw a figure in Canvas. +// - If isRandom == true, the figure will redrawn everytime Draw() is called, +// - If isRandom == false and Draw() was called before, it won't redraw. +func (ii *IdentIcon) Draw() { + + if ii.hasBeenDrawn && !ii.isRandom { + // Don't redraw once twice unless isRandom is enabled. + return + } else if ii.isRandom { + // Set a new randomSeed everytime Draw is executed to produce different + // results on each execution. + ii.randomSeed = strconv.Itoa(ii.rand.Int()) + } + + ii.hasBeenDrawn = true + + // Make sure that the canvas has been initialized. + ii.initCanvas() + + // current index of the digested bytes array. + var i int + // Number of bytes readed. + var readedBytes int + // Flag to know whether it as completed a full cycle. + var hasCompletedCycle bool + // Position that represents a point in the canvas. + var current image.Point + + // Text:Namespace:randomSeed + generatingBytes := []byte(ii.GeneratorText()) + + // Produce fixed-length array of bytes that will be used to control the + // drawing process. + hashBytes := ii.hashFunction(generatingBytes) + hashBytesLen := len(hashBytes) + + ii.FillColor = ii.fillColorFunction(hashBytes) + ii.BackgroundColor = ii.backgroundColorFunction(hashBytes, ii.FillColor) + + // Total number of iterations over the digested hash. + bytesToRead := ii.Density * ii.Size + + for { + if hasCompletedCycle { + // If the number of bytes to read exceeds the length of the hash, + // it will cycle through it. After it has completed a whole cycle, + // altering the value will produce more varied figures. + // + // XOR pseudo-random produces interesting results. + hashBytes[i] ^= byte(ii.rand.Intn(255)) + } + + if i == 0 { + // Everytime a new cycle is starting, change the current point to + // cover multiple areas of the canvas. + current = initialPoint( + hashBytes[0], + ii.rand.Intn(ii.drawableWidth), + ii.rand.Intn(ii.Size), + ) + } + + // value to add in the current point, zeroes will be ignored. + value := getFillValue(hashBytes[i]) + + if value != 0 { + // Initialize the map for Y-axis, making sure that the map that + // contains X-axis values won't be nil. + createMapIfDoesntExist(&ii.Canvas, current.Y) + + firstTimeFilled := false + if ii.Canvas.PointsMap[current.Y][current.X] == 0 { + // Increment FilledPoints the first time this point is visited. + ii.Canvas.FilledPoints++ + firstTimeFilled = true + } + + // Add the value to current position + ii.Canvas.PointsMap[current.Y][current.X] += value + + // Mark Y value as visited. This will be helpful to determine big + // blank spaces in the resulting figure. + ii.Canvas.VisitedYPoints[current.Y] = true + + // Update the maximum and minimum Y-axis values, useful to + // vertically center the figure at image creation. + if current.Y < ii.Canvas.MinY { + ii.Canvas.MinY = current.Y + } + if current.Y > ii.Canvas.MaxY { + ii.Canvas.MaxY = current.Y + } + + // When Size is an odd number, prevent points in the middle to be + // added twice. By substrating oddDiff to drawableWidth we make sure + // that it doesn't happens. + oddDiff := ii.Size % 2 + if current.X < (ii.drawableWidth - oddDiff) { + // Calculate the mirror position for X-axis + mirror := mirrorSymmetric(current, ii.Size) + // Add value to the mirrowed position + ii.Canvas.PointsMap[mirror.Y][mirror.X] += value + if firstTimeFilled { + ii.Canvas.FilledPoints++ + } + } + } + + // Decide the next position relative to the current position. + current = nextPoint(hashBytes[i], current, ii.drawableWidth, ii.Size) + + i++ + readedBytes++ + + if readedBytes >= bytesToRead { + // The total number of bytes to read has been reached, stop. + break + } + + if i == hashBytesLen-1 { + // A full cycle has been completed, reset the index to prevent + // getting out of bounds. + i = 0 + // Further iterations will add a pesudo-random number to hashBytes. + hasCompletedCycle = true + } + } + +} + +// GeneratorText returns the string later to be hashed using the format: +// - Text[:Namespace][:randomSeed] +func (ii *IdentIcon) GeneratorText() string { + gt := ii.Text + + if ii.Namespace != "" { + gt += ":" + ii.Namespace + } + + if ii.isRandom && ii.randomSeed != "" { + gt += ":" + ii.randomSeed + } + + return gt +} + +// Array generates a two-dimensional array version of the IdentIcon figure. +func (ii *IdentIcon) Array() [][]int { + return ii.Canvas.Array() +} + +// ToString generates a string version of the IdentIcon figure. +func (ii *IdentIcon) String(separator string, fillEmptyWith string) string { + return ii.Canvas.String(separator, fillEmptyWith) +} + +// Points generates an array of points of a two-dimensional plane as [x, y] +// that correspond to all filled points in the IdentIcon figure. +func (ii *IdentIcon) Points() []image.Point { + return ii.Canvas.Points() +} + +// IntCoordinates generates an array of points of a two-dimensional plane as: +// - [x, y] that correspond to all filled points in the IdentIcon figure. +func (ii *IdentIcon) IntCoordinates() [][]int { + return ii.Canvas.IntCoordinates() +} + +// New returns a pointer to IdentIcon. +func newIdentIcon( + text string, + namespace string, + size int, + density int, + isRandom bool, + rand *rand.Rand, + hashFunction func([]byte) []byte, + fillColorFunction func([]byte) color.Color, + backgroundColorFunction func([]byte, color.Color) color.Color, +) (*IdentIcon, error) { + + if text == "" { + // Text is the minimum requirement to generate an IdentIcon. + return nil, errors.New("Text can't be empty") + } + + if size < MinSize { + // Smaller values will generate a meaningless Generator. + return nil, errors.New( + "Size cannot be less than " + strconv.Itoa(MinSize), + ) + } + + if density < 1 { + return nil, errors.New( + "Density cannot be less than 1", + ) + } + + identicon := IdentIcon{ + Text: text, + Namespace: namespace, + Size: size, + Density: density, + isRandom: isRandom, + rand: rand, + hashFunction: hashFunction, + fillColorFunction: fillColorFunction, + backgroundColorFunction: backgroundColorFunction, + } + + // Reflection Line + identicon.drawableWidth = identicon.Size / 2 + + // Since the canvas is a symmetrical reflection make sure to: + // - Handle even and odd Canvas sizes + if identicon.Size%2 == 1 { + // Is odd, the vertical middle point exist. + identicon.drawableWidth++ + } + + return &identicon, nil +} + +// initCanvas initializes and erases everything that was in the Canvas map. +func (ii *IdentIcon) initCanvas() { + ii.Canvas = Canvas{ + Size: ii.Size, + PointsMap: make(map[int]map[int]int), + MinY: ii.Size, + MaxY: 0, + VisitedYPoints: make(map[int]bool), + } +} + +func nextPoint(control byte, p image.Point, width, heigth int) image.Point { + // Active bits will decide the destination of the next point. + // - If two opposite bits are active, it will keep its current position. + if control&moveUp == moveUp { + p.Y-- + } + if control&moveDown == moveDown { + p.Y++ + } + if control&moveLeft == moveLeft { + p.X-- + } + if control&moveRight == moveRight { + p.X++ + } + + // Transform to 0-based indices. + width-- + heigth-- + + // Teleport to opposite bounds when the limit has been reached. + if p.X > width { + p.X = 0 + } else if p.X < 0 { + p.X = width + } + if p.Y > heigth { + p.Y = 0 + } else if p.Y < 0 { + p.Y = heigth + } + + return p +} + +func initialPoint(control byte, width, heigth int) image.Point { + return image.Point{ + Y: heigth, + X: width, + } +} + +func mirrorSymmetric(p image.Point, size int) image.Point { + return image.Point{ + Y: p.Y, + X: size - p.X - 1, + } +} + +func getFillValue(control byte) int { + if control&fillPoint > 0 { + return 1 + } + return 0 +} + +func createMapIfDoesntExist(canvas *Canvas, y int) { + _, exist := canvas.PointsMap[y] + if !exist { + canvas.PointsMap[y] = make(map[int]int) + } +} diff --git a/backend/vendor/github.com/nullrocks/identicon/image.go b/backend/vendor/github.com/nullrocks/identicon/image.go new file mode 100644 index 00000000..c6d81a19 --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/image.go @@ -0,0 +1,73 @@ +package identicon + +import ( + "image" + "image/draw" + "image/jpeg" + "image/png" + "io" +) + +// Image genetares an image.Image of size +func (ii *IdentIcon) Image(pixels int) image.Image { + + // Padding is relative to the number of blocks. + padding := pixels / (ii.Size * MinSize) + drawableArea := pixels - (padding * 2) + blockSize := drawableArea / ii.Size + + // Add the residue (pixels that won't be filled) to the padding. + // Try to center the figure regardless when the drawable area is not + // divisible by the block pixels. + padding += (drawableArea % ii.Size) / 2 + + img := image.NewNRGBA(image.Rect(0, 0, pixels, pixels)) + + // Background + draw.Draw( + img, + img.Bounds(), + &image.Uniform{ii.BackgroundColor}, + image.ZP, + draw.Src, + ) + + for y, mapX := range ii.Canvas.PointsMap { + for x := range mapX { + + ix := blockSize*x + padding + iy := blockSize*y + padding + + draw.Draw(img, + image.Rect( + ix, + iy, + ix+blockSize, + iy+blockSize, + ), + &image.Uniform{ii.FillColor}, + image.ZP, + draw.Src, + ) + } + } + + return img +} + +// Png writes an image of pixels +func (ii *IdentIcon) Png(pixels int, w io.Writer) error { + img := ii.Image(pixels) + return png.Encode(w, img) +} + +// Jpeg writes an image of pixels and quality +func (ii *IdentIcon) Jpeg(pixels int, quality int, w io.Writer) error { + img := ii.Image(pixels) + return jpeg.Encode(w, img, &jpeg.Options{Quality: quality}) +} + +// Svg writes an image of pixels +func (ii *IdentIcon) Svg(pixels int, w io.Writer) error { + return svgEncode(w, ii, pixels) +} diff --git a/backend/vendor/github.com/nullrocks/identicon/svg.go b/backend/vendor/github.com/nullrocks/identicon/svg.go new file mode 100644 index 00000000..11fa130a --- /dev/null +++ b/backend/vendor/github.com/nullrocks/identicon/svg.go @@ -0,0 +1,92 @@ +package identicon + +import ( + "image/color" + "io" + "strconv" + "text/template" +) + +type svgRect struct { + X int + Y int + Width int + Height int + FillColor string +} + +type svgTmpl struct { + Pixels int + BackgroundColor string + FillColor string + Rects []svgRect +} + +const svgTemplate = ` + + + +{{range $index, $r := .Rects}} + {{end}} +` + +func colorToRGBAString(c color.Color) string { + r, g, b, a := c.RGBA() + r >>= 8 + g >>= 8 + b >>= 8 + a >>= 8 + + rs := strconv.Itoa(int(r)) + gs := strconv.Itoa(int(g)) + bs := strconv.Itoa(int(b)) + as := strconv.Itoa(int(a)) + + return "rgba(" + rs + "," + gs + "," + bs + "," + as + ")" +} + +// Encode an IdentIcon to SVG +func svgEncode(w io.Writer, ii *IdentIcon, pixels int) error { + + // Padding is relative to the number of blocks. + padding := pixels / (ii.Size * MinSize) + drawableArea := pixels - (padding * 2) + blockSize := drawableArea / ii.Size + + // Add the residue (pixels that won't be filled) to the padding. + // Try to center the figure regardless when the drawable area is not + // divisible by the block pixels. + padding += (drawableArea % ii.Size) / 2 + + fillColor := colorToRGBAString(ii.FillColor) + backgroundColor := colorToRGBAString(ii.BackgroundColor) + + b, err := template.New("svg").Parse(svgTemplate) + + if err != nil { + return err + } + + t := svgTmpl{ + Pixels: pixels, + BackgroundColor: backgroundColor, + FillColor: fillColor, + Rects: make([]svgRect, ii.Canvas.FilledPoints), + } + + i := 0 + for y, mapX := range ii.Canvas.PointsMap { + for x := range mapX { + t.Rects[i] = svgRect{ + blockSize*x + padding, + blockSize*y + padding, + blockSize, + blockSize, + fillColor, + } + i++ + } + } + + return b.Execute(w, t) +} From 547ea8375e40dc6727fc90ca4e9d5474c9faab93 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 10 Jul 2018 17:05:03 -0500 Subject: [PATCH 03/93] change deletem result link to /web #73 --- backend/app/rest/api/rest_private.go | 2 +- backend/app/rest/api/rest_private_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 74c17fb6..5a7159cd 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -263,6 +263,6 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token") return } - link := fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", s.RemarkURL, tokenStr) + link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr) render.JSON(w, r, JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link}) } diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index 600f6bb8..0585b63c 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -375,7 +375,7 @@ func TestRest_DeleteMe(t *testing.T) { claims, err := srv.Authenticator.JWTService.Parse(token) assert.Nil(t, err) assert.Equal(t, "dev", claims.User.ID) - assert.Equal(t, "https://demo.remark42.com/api/v1/admin/deleteme?token="+token, m["link"]) + assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+token, m["link"]) req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil) assert.Nil(t, err) From 42b9baf0f39e3df7677a33101bb3c4c1bcc0792e Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 11 Jul 2018 13:42:00 -0500 Subject: [PATCH 04/93] respect REMARK_URL for dev callback #159 --- backend/app/rest/auth/dev_provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/rest/auth/dev_provider.go b/backend/app/rest/auth/dev_provider.go index 0d5b111f..7e4caef1 100644 --- a/backend/app/rest/auth/dev_provider.go +++ b/backend/app/rest/auth/dev_provider.go @@ -144,7 +144,7 @@ func NewDev(p Params) Provider { AuthURL: fmt.Sprintf("http://127.0.0.1:%d/login/oauth/authorize", devAuthPort), TokenURL: fmt.Sprintf("http://127.0.0.1:%d/login/oauth/access_token", devAuthPort), }, - RedirectURL: "http://127.0.0.1:8080/auth/dev/callback", + RedirectURL: p.RemarkURL + "/auth/dev/callback", Scopes: []string{"user:email"}, InfoURL: fmt.Sprintf("http://127.0.0.1:%d/user", devAuthPort), MapUser: func(data userData, _ []byte) store.User { @@ -165,7 +165,7 @@ func (d *DevAuthServer) genAvatar(user string) ([]byte, error) { ii, err := d.iconGen.Draw(user) // Generate an IdentIcon if err != nil { - return nil, errors.Wrapf(err, "failed to draqw avatar for %s", user) + return nil, errors.Wrapf(err, "failed to draw avatar for %s", user) } buf := &bytes.Buffer{} From 25c75011f40fa165ee23f93e43bc6e12f8650d5a Mon Sep 17 00:00:00 2001 From: Aleksey Gurianov Date: Wed, 11 Jul 2018 00:12:43 +0400 Subject: [PATCH 05/93] tweak CI detect on frontend --- Dockerfile | 2 +- web/webpack.config.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0c844e33..efa84d9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,7 @@ ARG CI RUN apk add --no-cache --update git ADD web/package.json /srv/web/package.json ADD web/package-lock.json /srv/web/package-lock.json -RUN cd /srv/web && npm ci +RUN cd /srv/web && CI=true npm ci FROM node:10.6-alpine as build-frontend diff --git a/web/webpack.config.js b/web/webpack.config.js index 444d693f..c2771a02 100644 --- a/web/webpack.config.js +++ b/web/webpack.config.js @@ -125,7 +125,7 @@ module.exports = { }), new webpack.optimize.ModuleConcatenationPlugin(), ...(env === 'production' ? [new webpack.optimize.UglifyJsPlugin()] : []), - ...(process.env.CI === 'true' + ...(process.env.CI ? [] : [ new BundleAnalyze({ From f74f9e462c48d42a4918ca5472cb7547d1db660b Mon Sep 17 00:00:00 2001 From: Aleksey Gurianov Date: Thu, 12 Jul 2018 00:20:05 +0400 Subject: [PATCH 06/93] improve ux of login flow #157 --- web/app/components/root/root.jsx | 14 +++++++++----- web/iframe.html | 5 +++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/web/app/components/root/root.jsx b/web/app/components/root/root.jsx index 9a9f479f..04713530 100644 --- a/web/app/components/root/root.jsx +++ b/web/app/components/root/root.jsx @@ -111,18 +111,22 @@ export default class Root extends Component { onSignIn(provider) { const newWindow = window.open( - `${BASE_URL}/auth/${provider}/login?from=${encodeURIComponent(location.href)}&site=${siteId}` + `${BASE_URL}/auth/${provider}/login?from=${encodeURIComponent( + location.origin + location.pathname + '?selfClose' + )}&site=${siteId}` ); let secondsPass = 0; - const checkMsDelay = 100; + const checkMsDelay = 300; const checkInterval = setInterval(() => { + let shouldProceed; secondsPass += checkMsDelay; + try { + shouldProceed = newWindow.closed || secondsPass > 30000; + } catch (e) {} - if (newWindow.location.origin === location.origin || secondsPass > 30000) { + if (shouldProceed) { clearInterval(checkInterval); - secondsPass = 0; - newWindow.close(); api .getUser() diff --git a/web/iframe.html b/web/iframe.html index bc7d0933..5c841c04 100644 --- a/web/iframe.html +++ b/web/iframe.html @@ -5,6 +5,11 @@ remark42 + - - -
-
-
- - - - diff --git a/web/iframe.html b/web/iframe.html index 5c841c04..d65d6c24 100644 --- a/web/iframe.html +++ b/web/iframe.html @@ -1,5 +1,5 @@ - + @@ -75,6 +75,13 @@ + + diff --git a/web/webpack.config.js b/web/webpack.config.js index c71449f3..cceb3a6f 100644 --- a/web/webpack.config.js +++ b/web/webpack.config.js @@ -46,6 +46,7 @@ module.exports = { counter: './app/counter', 'last-comments': './app/last-comments', remark: './app/remark', + deleteme: './app/deleteme', }, output: { path: publicFolder, @@ -129,7 +130,7 @@ module.exports = { openAnalyzer: false, }), ]), - new Copy(['./iframe.html']), + new Copy(['./iframe.html', './deleteme.html']), ], watchOptions: { ignored: /(node_modules|\.vendor\.js$)/, From b27489e9180e92ab65dc376ec1b914e2ccb9bfa7 Mon Sep 17 00:00:00 2001 From: Aleksey Gurianov Date: Tue, 17 Jul 2018 23:18:34 +0400 Subject: [PATCH 19/93] Logout user on deleteme request --- web/app/common/fetcher.js | 2 +- web/app/components/auth-panel/auth-panel.jsx | 2 +- web/app/utils/email.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/app/common/fetcher.js b/web/app/common/fetcher.js index 1e7b7b33..39285155 100644 --- a/web/app/common/fetcher.js +++ b/web/app/common/fetcher.js @@ -30,7 +30,7 @@ methods.forEach(method => { parameters.url = `${basename}${url}`; - if (method !== 'post' && !parameters.url.includes('?site=') && !parameters.url.includes('&site=')) { + if (siteId && method !== 'post' && !parameters.url.includes('?site=') && !parameters.url.includes('&site=')) { parameters.url += (parameters.url.includes('?') ? '&' : '?') + `site=${siteId}`; } diff --git a/web/app/components/auth-panel/auth-panel.jsx b/web/app/components/auth-panel/auth-panel.jsx index a3657625..2ed169dc 100644 --- a/web/app/components/auth-panel/auth-panel.jsx +++ b/web/app/components/auth-panel/auth-panel.jsx @@ -65,7 +65,7 @@ export default class AuthPanel extends Component { - diff --git a/web/app/utils/email.js b/web/app/utils/email.js index 5ad99850..c94e3913 100644 --- a/web/app/utils/email.js +++ b/web/app/utils/email.js @@ -27,7 +27,7 @@ link: ${link} } export function requestDeletion() { - deleteMe().then(data => { + return deleteMe().then(data => { const email = store.get('config').admin_email; const { subject, message } = getDeleteInformationMessage(data.user_id, siteId, data.link); window.location = `mailto:${email}?subject=${subject}&body=${message}`; From 10e79cdd906e65e08e66ca7d1c03d17c4e640f3c Mon Sep 17 00:00:00 2001 From: Aleksei Gurianov Date: Wed, 18 Jul 2018 22:27:56 +0400 Subject: [PATCH 20/93] another flaky ttl block test (#172) see #120 --- backend/app/rest/api/admin_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index 08d36884..7193a120 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -204,7 +204,7 @@ func TestAdmin_Block(t *testing.T) { assert.Equal(t, false, j["block"]) // block with ttl - code, _ = block(1, "10ms") + code, _ = block(1, "50ms") require.Equal(t, 200, code) res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time") @@ -216,7 +216,7 @@ func TestAdmin_Block(t *testing.T) { assert.Equal(t, "", comments.Comments[0].Text) assert.True(t, comments.Comments[0].Deleted) - time.Sleep(11 * time.Millisecond) + time.Sleep(50 * time.Millisecond) res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time") assert.Equal(t, 200, code) comments = commentsWithInfo{} From 8f3b8e11e13d91d39293959b418ba7b49b61a969 Mon Sep 17 00:00:00 2001 From: Aleksey Gurianov Date: Wed, 18 Jul 2018 23:37:00 +0400 Subject: [PATCH 21/93] Fix crossdomain clickoutside listener --- web/app/components/dropdown/dropdown.jsx | 23 +++++++++++++++++------ web/app/deleteme.js | 1 + web/app/embed.js | 9 +++++++-- web/webpack.config.js | 2 +- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/web/app/components/dropdown/dropdown.jsx b/web/app/components/dropdown/dropdown.jsx index 439d11df..c28773f5 100644 --- a/web/app/components/dropdown/dropdown.jsx +++ b/web/app/components/dropdown/dropdown.jsx @@ -13,6 +13,7 @@ export default class Dropdown extends Component { this.onTitleClick = this.onTitleClick.bind(this); this.onOutsideClick = this.onOutsideClick.bind(this); + this.receiveMessage = this.receiveMessage.bind(this); } onTitleClick() { @@ -25,6 +26,20 @@ export default class Dropdown extends Component { } } + receiveMessage(e) { + try { + const data = typeof e.data === 'string' ? JSON.parse(e.data) : e.data; + + if (data.clickOutside) { + if (this.state.isActive) { + this.setState({ + isActive: false, + }); + } + } + } catch (e) {} + } + onOutsideClick(e) { if (!this.rootNode.contains(e.target)) { if (this.state.isActive) { @@ -38,17 +53,13 @@ export default class Dropdown extends Component { componentDidMount() { document.addEventListener('click', this.onOutsideClick); - if (parent) { - parent.document.addEventListener('click', this.onOutsideClick); - } + window.addEventListener('message', this.receiveMessage); } componentWillUnmount() { document.removeEventListener('click', this.onOutsideClick); - if (parent) { - parent.document.removeEventListener('click', this.onOutsideClick); - } + window.removeEventListener('message', this.receiveMessage); } render(props, { isActive }) { diff --git a/web/app/deleteme.js b/web/app/deleteme.js index 2186461c..e3c96194 100644 --- a/web/app/deleteme.js +++ b/web/app/deleteme.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ import { NODE_ID } from 'common/constants'; import { approveDeleteMe } from 'common/api'; import { token } from 'common/settings'; diff --git a/web/app/embed.js b/web/app/embed.js index 17a7eae3..efc07637 100644 --- a/web/app/embed.js +++ b/web/app/embed.js @@ -51,9 +51,8 @@ function init() { const iframe = node.getElementsByTagName('iframe')[0]; window.addEventListener('message', receiveMessages); - window.addEventListener('hashchange', postHashToIframe); - + document.addEventListener('click', postClickOutsideToIframe); setTimeout(postHashToIframe, 1000); const remarkRootId = 'remark-km423lmfdslkm34'; @@ -230,4 +229,10 @@ function init() { iframe.contentWindow.postMessage(JSON.stringify({ hash }), '*'); } } + + function postClickOutsideToIframe(e) { + if (!iframe.contains(e.target)) { + iframe.contentWindow.postMessage(JSON.stringify({ clickOutside: true }), '*'); + } + } } diff --git a/web/webpack.config.js b/web/webpack.config.js index cceb3a6f..b73ddb28 100644 --- a/web/webpack.config.js +++ b/web/webpack.config.js @@ -17,7 +17,7 @@ const remarkUrl = process.env.REMARK_URL || 'https://demo.remark42.com'; const NODE_ID = 'remark42'; // let's log some env variables because we can console.log(`NODE_ENV = ${env}`); -console.log(`REMARK_ENV = ${process.env.REMARK_URL}`); +console.log(`REMARK_ENV = ${remarkUrl}`); const commonStyleLoaders = [ 'css-loader', From c1e26167641c299132fc6e3b898c034e2678a504 Mon Sep 17 00:00:00 2001 From: Alexandr K Date: Thu, 19 Jul 2018 01:16:37 +0500 Subject: [PATCH 22/93] Input fix for Edge issue 169 (#171) --- web/app/components/input/input.jsx | 50 +++++-------------- .../components/input/textarea-autosize.jsx | 34 +++++++++++++ 2 files changed, 46 insertions(+), 38 deletions(-) create mode 100644 web/app/components/input/textarea-autosize.jsx diff --git a/web/app/components/input/input.jsx b/web/app/components/input/input.jsx index b8e28069..47f57763 100644 --- a/web/app/components/input/input.jsx +++ b/web/app/components/input/input.jsx @@ -6,6 +6,7 @@ import { siteId, url } from 'common/settings'; import api from 'common/api'; import store from 'common/store'; +import TextareaAutosize from 'components/input/textarea-autosize'; const RSS_THREAD_URL = `${BASE_URL}${API_BASE}/rss/post?site=${siteId}&url=${url}`; const RSS_SITE_URL = `${BASE_URL}${API_BASE}/rss/site?site=${siteId}`; @@ -21,7 +22,7 @@ export default class Input extends Component { isErrorShown: false, isDisabled: false, maxLength: config.max_comment_size || DEFAULT_MAX_COMMENT_SIZE, - commentLength: 0, + text: props.value || '', }; this.send = this.send.bind(this); @@ -31,28 +32,11 @@ export default class Input extends Component { } componentDidMount() { - const { mods = {}, value } = this.props; - - if (this.props.autoFocus) { - this.fieldNode.focus(); - } - - if (mods.mode !== 'edit') { - this.fieldNode.value = ''; - } else { - this.fieldNode.value = value; - this.autoResize(); - } - store.onUpdate('config', config => { this.setState({ maxLength: (config && config.max_comment_size) || DEFAULT_MAX_COMMENT_SIZE }); }); } - componentWillUnmount() { - this.fieldNode.value = ''; - } - shouldComponentUpdate(nextProps, nextState) { return ( nextProps.id !== this.props.id || @@ -70,23 +54,16 @@ export default class Input extends Component { } } - onInput() { - this.autoResize(); - + onInput(e) { this.setState({ preview: null, isErrorShown: false, - commentLength: this.fieldNode.value.length, + text: e.target.value, }); } - autoResize() { - this.fieldNode.style.height = ''; - this.fieldNode.style.height = `${this.fieldNode.scrollHeight}px`; - } - send(e) { - const text = this.fieldNode.value; + const text = this.state.text; const { mods = {}, pid, id } = this.props; if (e) e.preventDefault(); @@ -104,9 +81,7 @@ export default class Input extends Component { this.props.onSubmit(comment); } - this.fieldNode.value = ''; - this.fieldNode.style.height = ''; - this.setState({ preview: null }); + this.setState({ preview: null, text: '' }); }) .catch(() => { this.setState({ isErrorShown: true }); @@ -115,7 +90,7 @@ export default class Input extends Component { } getPreview() { - const text = this.fieldNode.value; + const text = this.state.text; if (!text || !text.trim()) return; @@ -129,21 +104,20 @@ export default class Input extends Component { }); } - render(props, { isDisabled, isErrorShown, preview, maxLength, commentLength }) { - const charactersLeft = maxLength - commentLength; - const { mods = {}, value = null, errorMessage } = props; + render(props, { isDisabled, isErrorShown, preview, maxLength, text }) { + const charactersLeft = maxLength - text.length; + const { mods = {}, errorMessage } = props; return (
- + ); + } +} From 3abe4146b13d899fefbd01cf5ee9bbb5864483d0 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 18 Jul 2018 16:56:23 -0500 Subject: [PATCH 23/93] add deleteme flag to jwt token it should prevent misuse of laked token to request user's data removal --- backend/app/rest/api/admin.go | 5 +++++ backend/app/rest/api/admin_test.go | 17 +++++++++++++++++ backend/app/rest/api/rest_private.go | 4 +++- backend/app/rest/auth/auth.go | 6 ++++++ backend/app/rest/auth/auth_test.go | 22 ++++++++++++++++++++++ backend/app/rest/auth/jwt.go | 3 +++ 6 files changed, 56 insertions(+), 1 deletion(-) diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 56deb942..c1f5e760 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -108,6 +108,11 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.SiteID) + if !claims.DeleteMe { // deletme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token + rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token") + return + } + if err := a.dataService.DeleteUser(claims.SiteID, claims.User.ID); err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user") return diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index 7193a120..1e0b5a93 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -497,6 +497,7 @@ func TestAdmin_DeleteMeRequest(t *testing.T) { User: &store.User{ ID: "user1", }, + DeleteMe: true, } token, err := srv.Authenticator.JWTService.Token(&claims) @@ -551,6 +552,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) { User: &store.User{ ID: "user1", }, + DeleteMe: true, } token, err := srv.Authenticator.JWTService.Token(&claims) @@ -573,6 +575,21 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) { resp, err = client.Do(req) assert.Nil(t, err) assert.Equal(t, 400, resp.StatusCode, resp.Status) + + // try without deleteme flag + badClaims2 := claims + badClaims2.DeleteMe = false + token, err = srv.Authenticator.JWTService.Token(&badClaims2) + assert.Nil(t, err) + req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil) + assert.Nil(t, err) + req.SetBasicAuth("dev", "password") + resp, err = client.Do(req) + assert.Nil(t, err) + assert.Equal(t, 403, resp.StatusCode) + b, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.True(t, strings.Contains(string(b), "can't use provided token")) } func TestAdmin_GetUserInfo(t *testing.T) { diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 5a7159cd..8bbc0815 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -255,7 +255,8 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) { ExpiresAt: time.Now().AddDate(0, 3, 0).Unix(), NotBefore: time.Now().Add(-1 * time.Minute).Unix(), }, - User: &user, + User: &user, + DeleteMe: true, // prevent this token from being used for login } tokenStr, err := s.Authenticator.JWTService.Token(&claims) @@ -263,6 +264,7 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token") return } + link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr) render.JSON(w, r, JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link}) } diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index a8eb2ae8..f9a9f202 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -73,6 +73,12 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { return } + if claims.DeleteMe { + log.Printf("[DEBUG] invalid token flags for %s/%s", claims.User.Name, claims.User.ID) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + if a.JWTService.IsExpired(claims) { if claims, err = a.refreshExpiredToken(w, claims); err != nil { log.Printf("[DEBUG] can't refresh jwt, %s", err) diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index 1799e174..39194f36 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -15,6 +15,8 @@ import ( var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOnRydWV9LCJzdGF0ZSI6IjEyMzQ1NiIsImZyb20iOiJmcm9tIn0.6P_OwGf8CUJRtvNSlW20GmaMb5pFvCNemP94fHCqb5Q" +var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZGVsZXRlbWUiOnRydWV9.3wiT5fqDv_bzPky6-3IilU8ExfzCyvLpKDMPYOAFWEo" + func TestAuthJWTCookie(t *testing.T) { a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour), PermissionChecker: &mockUserPermissions{}} @@ -100,6 +102,26 @@ func TestAuthJWtBlocked(t *testing.T) { assert.Equal(t, 401, resp.StatusCode, "blocked user") } +func TestAuthJWtFlags(t *testing.T) { + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour)} + router := chi.NewRouter() + router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(201) + }) + server := httptest.NewServer(router) + defer server.Close() + + jar, err := cookiejar.New(nil) + require.Nil(t, err) + client := &http.Client{Jar: jar, Timeout: 5 * time.Second} + req, err := http.NewRequest("GET", server.URL+"/auth", nil) + require.Nil(t, err) + req.Header.Add("X-JWT", testJwtDeleteMe) + resp, err := client.Do(req) + require.NoError(t, err) + assert.Equal(t, 401, resp.StatusCode, "blocked user") +} + func TestAuthRequired(t *testing.T) { a := Authenticator{DevPasswd: "123456"} router := chi.NewRouter() diff --git a/backend/app/rest/auth/jwt.go b/backend/app/rest/auth/jwt.go index 68842ced..0b1673fe 100644 --- a/backend/app/rest/auth/jwt.go +++ b/backend/app/rest/auth/jwt.go @@ -29,6 +29,9 @@ type CustomClaims struct { From string `json:"from,omitempty"` SiteID string `json:"site_id,omitempty"` SessionOnly bool `json:"sess_only,omitempty"` + + // flags indicate different uses + DeleteMe bool `json:"deleteme,omitempty"` } const jwtCookieName = "JWT" From 0ae7f226af7ad287f361e07ef739cb53357cc602 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 18 Jul 2018 21:05:51 -0500 Subject: [PATCH 24/93] extend jwt claim with flags --- backend/app/rest/api/admin.go | 3 ++- backend/app/rest/api/admin_test.go | 6 +++--- backend/app/rest/api/rest_private.go | 4 ++-- backend/app/rest/auth/auth.go | 2 +- backend/app/rest/auth/auth_test.go | 2 +- backend/app/rest/auth/jwt.go | 10 +++++++++- backend/app/rest/auth/jwt_test.go | 10 +++------- backend/app/rest/auth/provider.go | 1 + 8 files changed, 22 insertions(+), 16 deletions(-) diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index c1f5e760..04baee07 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -108,7 +108,8 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.SiteID) - if !claims.DeleteMe { // deletme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token + // deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token + if !claims.Flags.DeleteMe { rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token") return } diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index 1e0b5a93..b8412d50 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -497,8 +497,8 @@ func TestAdmin_DeleteMeRequest(t *testing.T) { User: &store.User{ ID: "user1", }, - DeleteMe: true, } + claims.Flags.DeleteMe = true token, err := srv.Authenticator.JWTService.Token(&claims) assert.Nil(t, err) @@ -552,8 +552,8 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) { User: &store.User{ ID: "user1", }, - DeleteMe: true, } + claims.Flags.DeleteMe = true token, err := srv.Authenticator.JWTService.Token(&claims) assert.Nil(t, err) @@ -578,7 +578,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) { // try without deleteme flag badClaims2 := claims - badClaims2.DeleteMe = false + badClaims2.Flags.DeleteMe = false token, err = srv.Authenticator.JWTService.Token(&badClaims2) assert.Nil(t, err) req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil) diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 8bbc0815..d8465d82 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -255,9 +255,9 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) { ExpiresAt: time.Now().AddDate(0, 3, 0).Unix(), NotBefore: time.Now().Add(-1 * time.Minute).Unix(), }, - User: &user, - DeleteMe: true, // prevent this token from being used for login + User: &user, } + claims.Flags.DeleteMe = true // prevent this token from being used for login tokenStr, err := s.Authenticator.JWTService.Token(&claims) if err != nil { diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index f9a9f202..e66c2131 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -73,7 +73,7 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { return } - if claims.DeleteMe { + if a.JWTService.HasFlags(claims) { log.Printf("[DEBUG] invalid token flags for %s/%s", claims.User.Name, claims.User.ID) http.Error(w, "Unauthorized", http.StatusUnauthorized) return diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index 39194f36..e7a50d71 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -15,7 +15,7 @@ import ( var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOnRydWV9LCJzdGF0ZSI6IjEyMzQ1NiIsImZyb20iOiJmcm9tIn0.6P_OwGf8CUJRtvNSlW20GmaMb5pFvCNemP94fHCqb5Q" -var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZGVsZXRlbWUiOnRydWV9.3wiT5fqDv_bzPky6-3IilU8ExfzCyvLpKDMPYOAFWEo" +var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZmxhZ3MiOnsiZGVsZXRlbWUiOnRydWV9fQ.SLh1QpFytWZqcT99VgcdAOtgFKhvpKCcZwqWTvAd63g" func TestAuthJWTCookie(t *testing.T) { a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour), diff --git a/backend/app/rest/auth/jwt.go b/backend/app/rest/auth/jwt.go index 0b1673fe..54855432 100644 --- a/backend/app/rest/auth/jwt.go +++ b/backend/app/rest/auth/jwt.go @@ -31,7 +31,10 @@ type CustomClaims struct { SessionOnly bool `json:"sess_only,omitempty"` // flags indicate different uses - DeleteMe bool `json:"deleteme,omitempty"` + Flags struct { + Login bool `json:"login,omitempty"` + DeleteMe bool `json:"deleteme,omitempty"` + } `json:"flags,omitempty"` } const jwtCookieName = "JWT" @@ -60,6 +63,11 @@ func (j *JWT) Token(claims *CustomClaims) (string, error) { return tokenString, nil } +// HasFlags indicates presense of special flags +func (j *JWT) HasFlags(claims *CustomClaims) bool { + return claims.Flags.DeleteMe || claims.Flags.Login +} + // Parse token string and verify. Not checking for expiration func (j *JWT) Parse(tokenString string) (*CustomClaims, error) { parser := jwt.Parser{SkipClaimsValidation: true} // allow parsing of expired tokens diff --git a/backend/app/rest/auth/jwt_test.go b/backend/app/rest/auth/jwt_test.go index bb5eb084..0795809b 100644 --- a/backend/app/rest/auth/jwt_test.go +++ b/backend/app/rest/auth/jwt_test.go @@ -14,13 +14,9 @@ import ( "github.com/umputun/remark/backend/app/store" ) -var testJwtValid = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCI" + - "sImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZS" + - "I6IiIsImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ._loFgh3g45gr9TtGqvM3N584I_6EHEOJnYb6Py84stQ" +var testJwtValid = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlb" + "WFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZX0" + "sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJmbGFncyI6e319.E2Blxqo1wsY855q258c0obxFJ1lgJciv1av1ewzlJBs" -var testJwtValidSess = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6In" + - "JlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsIm" + - "FkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJzZXNzX29ubHkiOnRydWV9.p6w0sM_NYaRuyhyA9jqfWlB5cx1vZPGhXGC5geSX7nA" +var testJwtValidSess = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIs" + "ImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFk" + "bWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJzZXNzX29ubHkiOnRydWUsImZsYWdzIjp7fX0." + "nKhehF1Xiome1yK1ewfOiIsrATvq7Tx7p1BCSJqKHuo" var testJwtExpired = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1MjY4ODc4MjIsImp0aSI6InJhbmRvbSBpZCIs" + "ImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiI" + @@ -89,9 +85,9 @@ func TestJWT_Set(t *testing.T) { ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(), NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(), }, + SessionOnly: false, } - claims.SessionOnly = false rr := httptest.NewRecorder() err := j.Set(rr, claims, claims.SessionOnly) assert.Nil(t, err) diff --git a/backend/app/rest/auth/provider.go b/backend/app/rest/auth/provider.go index 40946429..30f87739 100644 --- a/backend/app/rest/auth/provider.go +++ b/backend/app/rest/auth/provider.go @@ -99,6 +99,7 @@ func (p Provider) loginHandler(w http.ResponseWriter, r *http.Request) { NotBefore: time.Now().Add(-1 * time.Minute).Unix(), }, } + claims.Flags.Login = true if err := p.JwtService.Set(w, &claims, false); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set jwt") From 7e00d40596502951e2e4165aab7acdf783c591a2 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 19 Jul 2018 00:08:13 -0500 Subject: [PATCH 25/93] simplify user parsing with MustGetUserInfo --- backend/app/rest/api/rest_private.go | 41 ++++++---------------------- backend/app/rest/user.go | 11 ++++++++ 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index d8465d82..5435f570 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -31,11 +31,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - user, err := rest.GetUserInfo(r) - if err != nil { // this not suppose to happen (handled by Auth), just dbl-check - rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") - return - } + user := rest.MustGetUserInfo(r) log.Printf("[DEBUG] create comment %+v", comment) comment.PrepareUntrusted() // clean all fields user not supposed to set @@ -43,7 +39,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) { comment.User.IP = strings.Split(r.RemoteAddr, ":")[0] comment.Orig = comment.Text // original comment text, prior to md render - if err = s.DataService.ValidateComment(&comment); err != nil { + if err := s.DataService.ValidateComment(&comment); err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment") return } @@ -93,17 +89,14 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - user, err := rest.GetUserInfo(r) - if err != nil { // this not suppose to happen (handled by Auth), just dbl-check - rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") - return - } + user := rest.MustGetUserInfo(r) locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} id := chi.URLParam(r, "id") log.Printf("[DEBUG] update comment %s", id) var currComment store.Comment + var err error if currComment, err = s.DataService.Get(locator, id); err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment") return @@ -134,12 +127,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { // GET /user?site=siteID - returns user info func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) { - user, err := rest.GetUserInfo(r) - if err != nil { - rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") - return - } - + user := rest.MustGetUserInfo(r) if siteID := r.URL.Query().Get("site"); siteID != "" { user.Verified = s.DataService.IsVerified(siteID, user.ID) } @@ -149,12 +137,7 @@ func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) { // PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) { - - user, err := rest.GetUserInfo(r) - if err != nil { - rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") - return - } + user := rest.MustGetUserInfo(r) locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} id := chi.URLParam(r, "id") log.Printf("[DEBUG] vote for comment %s", id) @@ -179,11 +162,7 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) { // GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") - user, err := rest.GetUserInfo(r) - if err != nil { - rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") - return - } + user := rest.MustGetUserInfo(r) userB, err := json.Marshal(&user) if err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info") @@ -241,11 +220,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) { // POST /deleteme?site_id=site - requesting delete of all user info // makes jwt with user info and sends it back as a part of json response func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) { - user, err := rest.GetUserInfo(r) - if err != nil { - rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info") - return - } + user := rest.MustGetUserInfo(r) siteID := r.URL.Query().Get("site") claims := auth.CustomClaims{ diff --git a/backend/app/rest/user.go b/backend/app/rest/user.go index 877d414c..486c4b63 100644 --- a/backend/app/rest/user.go +++ b/backend/app/rest/user.go @@ -3,6 +3,7 @@ package rest import ( "context" "errors" + "log" "net/http" "github.com/umputun/remark/backend/app/store" @@ -10,6 +11,16 @@ import ( type contextKey string +// MustGetUserInfo fails if can't extract user data from the request. +// should be called from authed controllers only +func MustGetUserInfo(r *http.Request) store.User { + user, err := GetUserInfo(r) + if err != nil { + log.Fatalf("[ERROR] %s", err) + } + return user +} + // GetUserInfo returns user from request context func GetUserInfo(r *http.Request) (user store.User, err error) { From b2775d92b0f81d4a10c614e725d5796ed2a1d5f2 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 19 Jul 2018 00:09:32 -0500 Subject: [PATCH 26/93] chnage flatlf to panic from MustGetUserInfo --- backend/app/rest/user.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/app/rest/user.go b/backend/app/rest/user.go index 486c4b63..98d0092c 100644 --- a/backend/app/rest/user.go +++ b/backend/app/rest/user.go @@ -3,7 +3,6 @@ package rest import ( "context" "errors" - "log" "net/http" "github.com/umputun/remark/backend/app/store" @@ -16,7 +15,7 @@ type contextKey string func MustGetUserInfo(r *http.Request) store.User { user, err := GetUserInfo(r) if err != nil { - log.Fatalf("[ERROR] %s", err) + panic(err) } return user } From 1a423a464bbdbe4c222f596584e3041652b54269 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 19 Jul 2018 00:25:51 -0500 Subject: [PATCH 27/93] add tests --- backend/app/rest/user_test.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/backend/app/rest/user_test.go b/backend/app/rest/user_test.go index b4da4566..1df6d396 100644 --- a/backend/app/rest/user_test.go +++ b/backend/app/rest/user_test.go @@ -8,7 +8,7 @@ import ( "github.com/umputun/remark/backend/app/store" ) -func TestGetUserInfo(t *testing.T) { +func TestUser_GetUserInfo(t *testing.T) { r, err := http.NewRequest("GET", "http://blah.com", nil) assert.Nil(t, err) _, err = GetUserInfo(r) @@ -19,3 +19,21 @@ func TestGetUserInfo(t *testing.T) { assert.Nil(t, err) assert.Equal(t, store.User{Name: "test", ID: "id"}, u) } + +func TestUSer_MustGetUserInfo(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Log("recovered from panic") + } + }() + + r, err := http.NewRequest("GET", "http://blah.com", nil) + assert.Nil(t, err) + _ = MustGetUserInfo(r) + assert.Fail(t, "should panic") + + r = SetUserInfo(r, store.User{Name: "test", ID: "id"}) + u := MustGetUserInfo(r) + assert.Nil(t, err) + assert.Equal(t, store.User{Name: "test", ID: "id"}, u) +} From fdf161850b737c39f6ea635474d5c1297c304c32 Mon Sep 17 00:00:00 2001 From: Aleksei Gurianov Date: Thu, 19 Jul 2018 18:33:40 +0400 Subject: [PATCH 28/93] Close DataStore on cleanup in tests (#173) Windows fs can't remove testDb until file still used. Therefore we need to close datastore on cleanup. --- backend/app/rest/api/admin_test.go | 26 ++++++++++---------- backend/app/rest/api/rest_private_test.go | 22 ++++++++--------- backend/app/rest/api/rest_public_test.go | 30 +++++++++++------------ backend/app/rest/api/rest_test.go | 7 +++--- backend/app/rest/api/rss_test.go | 8 +++--- 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index b8412d50..d3cc64ec 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -20,7 +20,7 @@ import ( func TestAdmin_Delete(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"}, Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}} @@ -51,7 +51,7 @@ func TestAdmin_Delete(t *testing.T) { func TestAdmin_DeleteUser(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"}, Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}} @@ -105,7 +105,7 @@ func TestAdmin_DeleteUser(t *testing.T) { func TestAdmin_Pin(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}} @@ -149,7 +149,7 @@ func TestAdmin_Pin(t *testing.T) { func TestAdmin_Block(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}} @@ -230,7 +230,7 @@ func TestAdmin_Block(t *testing.T) { func TestAdmin_BlockedList(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) client := http.Client{} @@ -272,7 +272,7 @@ func TestAdmin_BlockedList(t *testing.T) { func TestAdmin_ReadOnly(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}} @@ -318,7 +318,7 @@ func TestAdmin_ReadOnly(t *testing.T) { func TestAdmin_ReadOnlyWithAge(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}, @@ -360,7 +360,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) { func TestAdmin_Verify(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}} @@ -417,7 +417,7 @@ func TestAdmin_Verify(t *testing.T) { func TestAdmin_ExportStream(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -437,7 +437,7 @@ func TestAdmin_ExportStream(t *testing.T) { func TestAdmin_ExportFile(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -469,7 +469,7 @@ func TestAdmin_ExportFile(t *testing.T) { func TestAdmin_DeleteMeRequest(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}} @@ -518,7 +518,7 @@ func TestAdmin_DeleteMeRequest(t *testing.T) { func TestAdmin_DeleteMeRequestFailed(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}} @@ -595,7 +595,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) { func TestAdmin_GetUserInfo(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}} diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index 0585b63c..26d09935 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -19,7 +19,7 @@ import ( func TestRest_Create(t *testing.T) { srv, ts := prep(t) require.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) resp, err := post(t, ts.URL+"/api/v1/comment", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`) @@ -40,7 +40,7 @@ func TestRest_Create(t *testing.T) { func TestRest_CreateOldPost(t *testing.T) { srv, ts := prep(t) require.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) // make old, but not too old comment old := store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5), @@ -74,7 +74,7 @@ func TestRest_CreateOldPost(t *testing.T) { func TestRest_CreateTooBig(t *testing.T) { srv, ts := prep(t) require.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) longComment := fmt.Sprintf(`{"text": "%4001s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ") @@ -106,7 +106,7 @@ func TestRest_CreateRejected(t *testing.T) { srv, ts := prep(t) require.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) body := `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}` // try to create without auth @@ -117,7 +117,7 @@ func TestRest_CreateRejected(t *testing.T) { func TestRest_CreateAndGet(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) // create comment resp, err := post(t, ts.URL+"/api/v1/comment", @@ -149,7 +149,7 @@ func TestRest_CreateAndGet(t *testing.T) { func TestRest_Update(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -188,7 +188,7 @@ func TestRest_Update(t *testing.T) { func TestRest_UpdateNotOwner(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "xyz"}} @@ -220,7 +220,7 @@ func TestRest_UpdateNotOwner(t *testing.T) { func TestRest_Vote(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}} @@ -264,7 +264,7 @@ func TestRest_Vote(t *testing.T) { func TestRest_UserAllData(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) // write 3 comments user := store.User{ID: "dev", Name: "user name 1"} @@ -319,7 +319,7 @@ func TestRest_UserAllData(t *testing.T) { func TestRest_UserAllDataManyComments(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) user := store.User{ID: "dev", Name: "user name 1"} c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", @@ -353,7 +353,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) { func TestRest_DeleteMe(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) client := http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil) diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go index 832a98d1..31886ba1 100644 --- a/backend/app/rest/api/rest_public_test.go +++ b/backend/app/rest/api/rest_public_test.go @@ -19,7 +19,7 @@ import ( func TestRest_Ping(t *testing.T) { srv, ts := prep(t) require.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) res, code := get(t, ts.URL+"/api/v1/ping") assert.Equal(t, "pong", res) @@ -29,7 +29,7 @@ func TestRest_Ping(t *testing.T) { func TestRest_Preview(t *testing.T) { srv, ts := prep(t) require.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`) assert.Nil(t, err) @@ -42,7 +42,7 @@ func TestRest_Preview(t *testing.T) { func TestRest_PreviewWithMD(t *testing.T) { srv, ts := prep(t) require.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) text := ` # h1 @@ -70,7 +70,7 @@ BKT func TestRest_Find(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) _, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1") assert.Equal(t, 400, code, "nothing in") @@ -124,7 +124,7 @@ func TestRest_Find(t *testing.T) { func TestRest_FindAge(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5), Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}} @@ -156,7 +156,7 @@ func TestRest_FindAge(t *testing.T) { func TestRest_FindReadOnly(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1), Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}} @@ -198,7 +198,7 @@ func TestRest_FindReadOnly(t *testing.T) { func TestRest_Last(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -243,7 +243,7 @@ func TestRest_Last(t *testing.T) { func TestRest_FindUserComments(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -280,7 +280,7 @@ func TestRest_FindUserComments(t *testing.T) { func TestRest_UserInfo(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) body, code := getWithAuth(t, ts.URL+"/api/v1/user?site=radio-t") assert.Equal(t, 200, code) @@ -294,7 +294,7 @@ func TestRest_UserInfo(t *testing.T) { func TestRest_Count(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -324,7 +324,7 @@ func TestRest_Count(t *testing.T) { func TestRest_Counts(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -354,7 +354,7 @@ func TestRest_Counts(t *testing.T) { func TestRest_List(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} @@ -381,7 +381,7 @@ func TestRest_List(t *testing.T) { func TestRest_Config(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) body, code := get(t, ts.URL+"/api/v1/config?site=radio-t") assert.Equal(t, 200, code) @@ -401,7 +401,7 @@ func TestRest_Config(t *testing.T) { func TestRest_Info(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only @@ -439,7 +439,7 @@ func TestRest_Info(t *testing.T) { func TestRest_Robots(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) body, code := get(t, ts.URL+"/robots.txt") assert.Equal(t, 200, code) diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 1813e223..6dd65230 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -31,7 +31,7 @@ var getStartedHTML = "/tmp/getstarted.html" func TestRest_FileServer(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) body, code := get(t, ts.URL+"/web/test-remark.html") assert.Equal(t, 200, code) @@ -41,7 +41,7 @@ func TestRest_FileServer(t *testing.T) { func TestRest_GetStarted(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) err := ioutil.WriteFile(getStartedHTML, []byte("some html blah"), 0700) assert.Nil(t, err) @@ -157,8 +157,9 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string { return crResp["id"].(string) } -func cleanup(ts *httptest.Server) { +func cleanup(ts *httptest.Server, srv *Rest) { ts.Close() + srv.DataService.Close() os.Remove(testDb) os.Remove(testHTML) } diff --git a/backend/app/rest/api/rss_test.go b/backend/app/rest/api/rss_test.go index 8894dd76..3eaffe14 100644 --- a/backend/app/rest/api/rss_test.go +++ b/backend/app/rest/api/rss_test.go @@ -14,7 +14,7 @@ import ( func TestServer_RssPost(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) waitOnSecChange() @@ -55,7 +55,7 @@ func TestServer_RssPost(t *testing.T) { func TestServer_RssSite(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) waitOnSecChange() @@ -109,7 +109,7 @@ func TestServer_RssSite(t *testing.T) { func TestServer_RssWithReply(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) waitOnSecChange() @@ -161,7 +161,7 @@ func TestServer_RssWithReply(t *testing.T) { func TestServer_RssReplies(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) - defer cleanup(ts) + defer cleanup(ts, srv) waitOnSecChange() From dbd1d4069f7089e52e234891c22ceacd5eb57c18 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 22 Jul 2018 22:46:25 -0400 Subject: [PATCH 29/93] feature/ext-mongo (#177) * externalize mongo wrapper * remove mongo env from drone * export mongo test for coverage report --- .drone.yml | 6 +- .travis.yml | 2 +- Dockerfile | 9 +- backend/Gopkg.lock | 8 +- backend/app/main.go | 5 +- backend/app/store/avatar/gridfs.go | 2 +- backend/app/store/avatar/gridfs_test.go | 3 +- backend/app/store/engine/mongo.go | 2 +- .../app/store/engine/mongo/connection_test.go | 126 -------------- backend/app/store/engine/mongo/server_test.go | 93 ---------- backend/app/store/engine/mongo/writer_test.go | 162 ------------------ backend/app/store/engine/mongo_test.go | 2 +- .../github.com/go-pkgz/mongo/.gitignore | 12 ++ .../github.com/go-pkgz/mongo/.travis.yml | 23 +++ .../github.com/davecgh/go-spew/LICENSE | 15 ++ .../.vendor/github.com/globalsign/mgo/LICENSE | 25 +++ .../github.com/globalsign/mgo/bson/LICENSE | 25 +++ .../globalsign/mgo/internal/json/LICENSE | 27 +++ .../github.com/pmezard/go-difflib/LICENSE | 27 +++ .../github.com/stretchr/testify/LICENSE | 22 +++ .../vendor/github.com/go-pkgz/mongo/LICENSE | 21 +++ .../vendor/github.com/go-pkgz/mongo/README.md | 58 +++++++ .../github.com/go-pkgz}/mongo/connection.go | 2 +- .../github.com/go-pkgz}/mongo/server.go | 18 +- .../github.com/go-pkgz}/mongo/testing.go | 29 ++-- .../github.com/go-pkgz}/mongo/writer.go | 17 +- compose-dev-backend.yml | 2 +- 27 files changed, 311 insertions(+), 432 deletions(-) delete mode 100644 backend/app/store/engine/mongo/connection_test.go delete mode 100644 backend/app/store/engine/mongo/server_test.go delete mode 100644 backend/app/store/engine/mongo/writer_test.go create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.gitignore create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.travis.yml create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/davecgh/go-spew/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/bson/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/internal/json/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/pmezard/go-difflib/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/stretchr/testify/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/mongo/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/mongo/README.md rename backend/{app/store/engine => vendor/github.com/go-pkgz}/mongo/connection.go (96%) rename backend/{app/store/engine => vendor/github.com/go-pkgz}/mongo/server.go (85%) rename backend/{app/store/engine => vendor/github.com/go-pkgz}/mongo/testing.go (81%) rename backend/{app/store/engine => vendor/github.com/go-pkgz}/mongo/writer.go (91%) diff --git a/.drone.yml b/.drone.yml index 3c28025a..1ba6f730 100644 --- a/.drone.yml +++ b/.drone.yml @@ -16,13 +16,12 @@ pipeline: docker_master: image: plugins/docker repo: umputun/remark42 - secrets: [ docker_username, docker_password, mongo_remark_test ] + secrets: [ docker_username, docker_password] build_args: - DRONE=${DRONE} - DRONE_TAG=${DRONE_TAG} - DRONE_COMMIT=${DRONE_COMMIT} - DRONE_BRANCH=${DRONE_BRANCH} - build_args_from_env: [mongo_remark_test] tags: - ${DRONE_COMMIT_BRANCH/\//-} when: @@ -46,12 +45,11 @@ pipeline: docker_branch: image: plugins/docker repo: umputun/remark42 - secrets: [ docker_username, docker_password, mongo_remark_test ] + secrets: [ docker_username, docker_password ] build_args: - DRONE=${DRONE} - DRONE_COMMIT=${DRONE_COMMIT} - DRONE_BRANCH=${DRONE_BRANCH} - build_args_from_env: [mongo_remark_test] tags: - ${DRONE_COMMIT_BRANCH/\//-} when: diff --git a/.travis.yml b/.travis.yml index 0dd8f7e3..1c64345a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,5 +16,5 @@ script: --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG --build-arg TRAVIS_TAG=$TRAVIS_TAG - --build-arg MONGO_REMARK_TEST=$MONGO_REMARK_TEST + --build-arg MONGO_TEST=$MONGO_REMARK_TEST . diff --git a/Dockerfile b/Dockerfile index b2527561..9f039463 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,23 +19,23 @@ ARG DRONE_BRANCH ARG DRONE_PULL_REQUEST ARG SKIP_BACKEND_TEST -ARG MONGO_REMARK_TEST +ARG MONGO_TEST WORKDIR /go/src/github.com/umputun/remark/backend ADD backend /go/src/github.com/umputun/remark/backend # run tests RUN \ - if [ -f .mongo ] ; then export MONGO_REMARK_TEST=$(cat .mongo) ; fi && \ + if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ cd app && \ if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \ else echo "skip backend test" ; fi -RUN echo "mongo=${MONGO_REMARK_TEST}" >> /etc/hosts +RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts # linters RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \ - if [ -f .mongo ] ; then export MONGO_REMARK_TEST=$(cat .mongo) ; fi && \ + if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \ --enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \ --enable=deadcode --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./... ; \ @@ -43,6 +43,7 @@ RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \ # coverage report RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \ + if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ mkdir -p target && /script/coverage.sh ; \ else echo "skip backend coverage" ; fi diff --git a/backend/Gopkg.lock b/backend/Gopkg.lock index 9d919b2a..e512a56e 100644 --- a/backend/Gopkg.lock +++ b/backend/Gopkg.lock @@ -81,6 +81,12 @@ revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519" version = "v1.0.0" +[[projects]] + name = "github.com/go-pkgz/mongo" + packages = ["."] + revision = "9a09a574c336c6ae2338a65bbebed2baab2a713c" + version = "v1.0.0" + [[projects]] name = "github.com/golang/protobuf" packages = ["proto"] @@ -250,6 +256,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b6dd6c99dd3a4f5135f8b869063a2b24044494b2ed77442bfd50a45d096e414a" + inputs-digest = "e933250b2582d1ff7d2b165c4a8a934762677a2ebaaea743fb3c485868c6d793" solver-name = "gps-cdcl" solver-version = 1 diff --git a/backend/app/main.go b/backend/app/main.go index d7e7bfc5..343a4dba 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -11,9 +11,8 @@ import ( "syscall" "time" - "github.com/umputun/remark/backend/app/store/avatar" - "github.com/coreos/bbolt" + "github.com/go-pkgz/mongo" "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" "github.com/pkg/errors" @@ -23,8 +22,8 @@ import ( "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" + "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/engine/mongo" "github.com/umputun/remark/backend/app/store/service" ) diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index f18a79b9..33106f5d 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -7,10 +7,10 @@ import ( "log" "github.com/globalsign/mgo" + "github.com/go-pkgz/mongo" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/engine/mongo" ) // NewGridFS makes gridfs (mongo) avatar store diff --git a/backend/app/store/avatar/gridfs_test.go b/backend/app/store/avatar/gridfs_test.go index fe4da914..bcc6baad 100644 --- a/backend/app/store/avatar/gridfs_test.go +++ b/backend/app/store/avatar/gridfs_test.go @@ -6,10 +6,9 @@ import ( "testing" "github.com/globalsign/mgo" + "github.com/go-pkgz/mongo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store/engine/mongo" ) func TestGridFS_PutAndGet(t *testing.T) { diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine/mongo.go index 8cde07ae..a6e50833 100644 --- a/backend/app/store/engine/mongo.go +++ b/backend/app/store/engine/mongo.go @@ -5,11 +5,11 @@ import ( "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" + "github.com/go-pkgz/mongo" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/engine/mongo" ) // Mongo implements engine interface diff --git a/backend/app/store/engine/mongo/connection_test.go b/backend/app/store/engine/mongo/connection_test.go deleted file mode 100644 index 0d611647..00000000 --- a/backend/app/store/engine/mongo/connection_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package mongo - -import ( - "fmt" - "testing" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/hashicorp/go-multierror" - "github.com/stretchr/testify/assert" -) - -type testRecord struct { - Symbol string - Num int -} - -func TestConnection_WithCollection(t *testing.T) { - c, err := write(t) - if err != nil { - return - } - defer RemoveTestCollection(t, c) - - var res []testRecord - err = c.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(nil).All(&res) - }) - assert.Nil(t, err) - assert.Equal(t, 100, len(res)) - - err = c.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"symbol": "blah"}).All(&res) - }) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) - - r1 := testRecord{} - err = c.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"symbol": "blah"}).One(&r1) - }) - assert.Equal(t, mgo.ErrNotFound, err) - - c = NewConnection(c.server, "test", "bbbbbbbaaad") - err = c.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"symbol": "blah"}).One(&r1) - }) - assert.Equal(t, mgo.ErrNotFound, err) -} - -func TestConnection_WithCollectionNoDB(t *testing.T) { - c, err := write(t) - if err != nil { - return - } - defer RemoveTestCollection(t, c) - - var res []testRecord - err = c.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(nil).All(&res) - }) - assert.Nil(t, err) - assert.Equal(t, 100, len(res)) -} - -func TestConnection_WithDB(t *testing.T) { - c, err := write(t) - if err != nil { - return - } - defer RemoveTestCollection(t, c) - - var res []testRecord - err = c.WithCustomDB("test", func(dbase *mgo.Database) error { - return dbase.C(c.collection).Find(nil).All(&res) - }) - assert.Nil(t, err) - assert.Equal(t, 100, len(res)) - - err = c.WithDB(func(dbase *mgo.Database) error { - return dbase.C(c.collection).Find(nil).All(&res) - }) - assert.Nil(t, err) - assert.Equal(t, 100, len(res)) -} - -func TestCleanup(t *testing.T) { - c, err := write(t) - if err != nil { - return - } - var res []testRecord - err = c.WithCustomDB("test", func(dbase *mgo.Database) error { - return dbase.C(c.collection).Find(nil).All(&res) - }) - assert.Nil(t, err) - assert.Equal(t, 100, len(res)) - - RemoveTestCollections(t, c, c.collection) - err = c.WithCustomDB("test", func(dbase *mgo.Database) error { - return dbase.C(c.collection).Find(nil).All(&res) - }) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func write(t *testing.T) (*Connection, error) { - c, err := MakeTestConnection(t) - if err != nil { - return nil, err - } - err = c.WithCollection(func(coll *mgo.Collection) error { - errs := new(multierror.Error) - for i := 0; i < 100; i++ { - r := testRecord{ - Symbol: fmt.Sprintf("symb-%02d", i%5), - Num: i, - } - insertErr := coll.Insert(r) - assert.Nil(t, insertErr, fmt.Sprintf("insert %+v", r)) - errs = multierror.Append(errs, insertErr) - } - return errs.ErrorOrNil() - }) - return c, err -} diff --git a/backend/app/store/engine/mongo/server_test.go b/backend/app/store/engine/mongo/server_test.go deleted file mode 100644 index 1b83548a..00000000 --- a/backend/app/store/engine/mongo/server_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package mongo - -import ( - "os" - "testing" - "time" - - "github.com/globalsign/mgo" - "github.com/stretchr/testify/assert" -) - -func TestServer_NewServerGood(t *testing.T) { - mongoURL := os.Getenv("MONGO_REMARK_TEST") - if mongoURL == "" || mongoURL == "skip" { - t.Skip("no MONGO_REMARK_TEST in env") - return - } - - m, err := NewServerWithURL(mongoURL, 3*time.Second) - assert.Nil(t, err) - assert.NotNil(t, m) - assert.True(t, m.String() != "") -} - -func TestServer_NewServerBad(t *testing.T) { - _, err := NewServerWithURL("mongodb://127.0.0.3:27017/test", 100*time.Millisecond) - assert.NotNil(t, err) - t.Log(err) - - _, err = NewServer(mgo.DialInfo{Addrs: []string{"127.0.0.2"}, Timeout: 100 * time.Millisecond}, ServerParams{}) - assert.NotNil(t, err) - - _, err = NewServer(mgo.DialInfo{}, ServerParams{}) - assert.NotNil(t, err) - - _, err = NewServerWithURL("mongodb://mongo:27017/test?blah=xxx", 100*time.Millisecond) - assert.NotNil(t, err) - t.Log(err) -} - -func TestServer_parse(t *testing.T) { - tbl := []struct { - mongoURL string - timeout time.Duration - params ServerParams - dial mgo.DialInfo - isErr bool - }{ - { - "mongodb://127.0.0.3:27017/test", time.Millisecond, - ServerParams{ConsistencyMode: 1}, - mgo.DialInfo{Addrs: []string{"127.0.0.3:27017"}, Timeout: 1000000, Database: "test", - ReadPreference: &mgo.ReadPreference{Mode: 2}}, - false, - }, - { - "mongodb://user:passwd@127.0.0.3:27017/test?ssl=true&authSource=admin", time.Millisecond, - ServerParams{ConsistencyMode: 1, SSL: true}, - mgo.DialInfo{Addrs: []string{"127.0.0.3:27017"}, Timeout: 1000000, Database: "test", Source: "admin", - Username: "user", Password: "passwd", ReadPreference: &mgo.ReadPreference{Mode: 2}}, - false, - }, - { - "mongodb://127.0.0.3", time.Millisecond, - ServerParams{ConsistencyMode: 1, SSL: false}, - mgo.DialInfo{Addrs: []string{"127.0.0.3"}, Timeout: 1000000, ReadPreference: &mgo.ReadPreference{Mode: 2}}, - false, - }, - { - "127.0.0.3", time.Millisecond, - ServerParams{ConsistencyMode: 1, SSL: false}, - mgo.DialInfo{Addrs: []string{"127.0.0.3"}, Timeout: 1000000, ReadPreference: &mgo.ReadPreference{Mode: 2}}, - false, - }, - { - "127.0.0.3?xxx=yyy", time.Millisecond, - ServerParams{}, mgo.DialInfo{}, - true, - }, - } - - for i, tt := range tbl { - dial, params, err := parseURL(tt.mongoURL, tt.timeout) - dial.DialServer = nil - if tt.isErr { - assert.NotNil(t, err, "expect error #%d", i) - t.Logf("dial %+v, params %+v", dial, params) - continue - } - assert.Equal(t, tt.dial, dial, "test #%d", i) - assert.Equal(t, tt.params, params, "test #%d", i) - } -} diff --git a/backend/app/store/engine/mongo/writer_test.go b/backend/app/store/engine/mongo/writer_test.go deleted file mode 100644 index 1c4a79f5..00000000 --- a/backend/app/store/engine/mongo/writer_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package mongo - -import ( - "sync" - "testing" - "time" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWriter(t *testing.T) { - - count := func(conn *Connection) (res int) { - _ = conn.WithCollection(func(coll *mgo.Collection) error { - var err error - res, err = coll.Find(nil).Count() - assert.Nil(t, err) - return nil - }) - return res - } - - conn, err := MakeTestConnection(t) - if err != nil { - return - } - defer RemoveTestCollection(t, conn) - - var wr BufferedWriter = NewBufferedWriter(3, conn) - assert.Nil(t, wr.Write(bson.M{"key1": "val1"}), "write rec #1") - assert.Nil(t, wr.Write(bson.M{"key2": "val2"}), "write rec #2") - - assert.Equal(t, 0, count(conn), "nothing yet") - - assert.Nil(t, wr.Write(bson.M{"key3": "val3"}), "write rec #3") - assert.Equal(t, 3, count(conn), "all 3 records in") - - assert.Nil(t, wr.Write(bson.M{"key4": "val4"}), "write rec #4") - assert.Equal(t, 3, count(conn), "still 3 records") - - assert.Nil(t, wr.Flush()) - assert.Equal(t, 4, count(conn), "all 4 records") - - assert.Nil(t, wr.Flush()) - assert.Equal(t, 4, count(conn), "still 4 records, nothing left to flush") - - assert.Nil(t, wr.Close()) - -} - -func TestWriter_Parallel(t *testing.T) { - conn, err := MakeTestConnection(t) - if err != nil { - return - } - defer RemoveTestCollection(t, conn) - - var wg sync.WaitGroup - wr := NewBufferedWriter(75, conn) - - writeMany := func() { - for i := 0; i < 100; i++ { - require.NoError(t, wr.Write(bson.M{"key1": 1, "key2": 2})) - } - wr.Flush() - wg.Done() - } - - for i := 0; i < 16; i++ { - wg.Add(1) - go writeMany() - } - - wg.Wait() - - _ = conn.WithCollection(func(coll *mgo.Collection) error { - res, err := coll.Find(nil).Count() - assert.Nil(t, err) - assert.Equal(t, 100*16, res) - return nil - }) - assert.Nil(t, wr.Close()) -} - -func TestWriter_WithAuthFlush(t *testing.T) { - conn, err := MakeTestConnection(t) - if err != nil { - return - } - defer RemoveTestCollection(t, conn) - - var wr BufferedWriter = NewBufferedWriter(3, conn).WithAutoFlush(500 * time.Millisecond) - count := func() (res int) { - _ = conn.WithCollection(func(coll *mgo.Collection) error { - var err error - res, err = coll.Find(nil).Count() - assert.Nil(t, err) - return nil - }) - return res - } - - assert.Nil(t, wr.Write(bson.M{"key1": "val1"}), "write rec #1") - assert.Nil(t, wr.Write(bson.M{"key2": "val2"}), "write rec #2") - assert.Equal(t, 0, count(), "nothing yet") - time.Sleep(600 * time.Millisecond) - assert.Equal(t, 2, count(), "2 records flushed") - - assert.Nil(t, wr.Write(bson.M{"key3": "val3"}), "write rec #3") - assert.Nil(t, wr.Write(bson.M{"key4": "val4"}), "write rec #4") - assert.Nil(t, wr.Write(bson.M{"key5": "val5"}), "write rec #5") - assert.Equal(t, 5, count(), "5 records, flushed by size, not duration") - - assert.Nil(t, wr.Write(bson.M{"key6": "val6"}), "write rec #6") - assert.Nil(t, wr.Write(bson.M{"key7": "val7"}), "write rec #7") - assert.Equal(t, 5, count(), "still 5 records") - - assert.Nil(t, wr.Flush()) - assert.Equal(t, 7, count(), "all 7 records") - - assert.Nil(t, wr.Flush()) - assert.Equal(t, 7, count(), "still 7 records, nothing left to flush") - assert.Nil(t, wr.Close()) -} - -func TestWriter_ParallelWithAutoFlush(t *testing.T) { - conn, err := MakeTestConnection(t) - if err != nil { - return - } - defer RemoveTestCollection(t, conn) - - var wg sync.WaitGroup - wr := NewBufferedWriter(75, conn).WithAutoFlush(time.Millisecond) - - writeMany := func() { - for i := 0; i < 100; i++ { - require.NoError(t, wr.Write(bson.M{"key1": 1, "key2": 2})) - time.Sleep(time.Millisecond * 3) - } - wr.Flush() - wg.Done() - } - - for i := 0; i < 16; i++ { - wg.Add(1) - go writeMany() - } - - wg.Wait() - - _ = conn.WithCollection(func(coll *mgo.Collection) error { - res, err := coll.Find(nil).Count() - assert.Nil(t, err) - assert.Equal(t, 100*16, res) - return nil - }) - assert.Nil(t, wr.Close()) -} diff --git a/backend/app/store/engine/mongo_test.go b/backend/app/store/engine/mongo_test.go index c2659c6d..0d94745b 100644 --- a/backend/app/store/engine/mongo_test.go +++ b/backend/app/store/engine/mongo_test.go @@ -6,11 +6,11 @@ import ( "testing" "time" + "github.com/go-pkgz/mongo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/engine/mongo" ) func TestMongo_CreateAndFind(t *testing.T) { diff --git a/backend/vendor/github.com/go-pkgz/mongo/.gitignore b/backend/vendor/github.com/go-pkgz/mongo/.gitignore new file mode 100644 index 00000000..f1c181ec --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.gitignore @@ -0,0 +1,12 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out diff --git a/backend/vendor/github.com/go-pkgz/mongo/.travis.yml b/backend/vendor/github.com/go-pkgz/mongo/.travis.yml new file mode 100644 index 00000000..6e8c06a7 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.travis.yml @@ -0,0 +1,23 @@ +language: go + +go: + - "1.10.x" + +go_import_path: github.com/go-pkgz/mongo + +services: mongodb + +before_install: + - cp -rf .vendor vendor + - go get github.com/mattn/goveralls + - go get gopkg.in/alecthomas/gometalinter.v2 + - $GOPATH/bin/gometalinter.v2 --install + +before_script: + - sleep 3 + +script: + - export MONGO_TEST=mongodb://127.0.0.1:27017 + - go test ./... + - $GOPATH/bin/gometalinter.v2 --exclude=test --exclude=mock --exclude=vendor ./... + - $GOPATH/bin/goveralls -service=travis-ci diff --git a/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/davecgh/go-spew/LICENSE b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/davecgh/go-spew/LICENSE new file mode 100644 index 00000000..c8364161 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/davecgh/go-spew/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/LICENSE b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/LICENSE new file mode 100644 index 00000000..770c7672 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/LICENSE @@ -0,0 +1,25 @@ +mgo - MongoDB driver for Go + +Copyright (c) 2010-2013 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/bson/LICENSE b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/bson/LICENSE new file mode 100644 index 00000000..89032601 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/bson/LICENSE @@ -0,0 +1,25 @@ +BSON library for Go + +Copyright (c) 2010-2012 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/internal/json/LICENSE b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/internal/json/LICENSE new file mode 100644 index 00000000..74487567 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/internal/json/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/pmezard/go-difflib/LICENSE b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/pmezard/go-difflib/LICENSE new file mode 100644 index 00000000..c67dad61 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/pmezard/go-difflib/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013, Patrick Mezard +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + The names of its contributors may not be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/stretchr/testify/LICENSE b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/stretchr/testify/LICENSE new file mode 100644 index 00000000..473b670a --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/stretchr/testify/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell + +Please consider promoting this project if you find it useful. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT +OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/mongo/LICENSE b/backend/vendor/github.com/go-pkgz/mongo/LICENSE new file mode 100644 index 00000000..ca125214 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Umputun + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/mongo/README.md b/backend/vendor/github.com/go-pkgz/mongo/README.md new file mode 100644 index 00000000..a3cd4c20 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/mongo/README.md @@ -0,0 +1,58 @@ +# Mongo [![Build Status](https://travis-ci.org/go-pkgz/mongo.svg?branch=master)](https://travis-ci.org/go-pkgz/mongo) [![Go Report Card](https://goreportcard.com/badge/github.com/go-pkgz/mongo)](https://goreportcard.com/report/github.com/go-pkgz/mongo) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/mongo/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/mongo?branch=master) + +Provides helpers on top of [mgo](https://github.com/globalsign/mgo) + +## Install and update + +`go get -u github.com/go-pkgz/mongo` + + +## Usage + +- `Server` represents mongo instance and provides session accessor. Application usually creates one server object and uses it for anything needed with this particular mongo host or replica set. + +- `Connection` encapsulates session and provides auto-closable wrapper. Each requests runs inside one of With* function makes new mongo session and closes on completion. + +- `BufferedWriter` implements buffered writer to mongo. Write method caching internally till it reached buffer size. Flush methods can be called manually at any time. + + +```golang + m, err := NewServerWithURL("mongodb://127.0.0.1:27017/test?debug=true", 3*time.Second) + if err != nil { + panic("can't make mongo server") + } + + type testRecord struct { + Key1 string + Kay2 int + } + + err = c.WithCollection(func(coll *mgo.Collection) error { // create session + // insert 100 records + for i := 0; i < 100; i++ { + r := testRecord{ + Key1: fmt.Sprintf("key-%02d", i%5), + Key2: i, + } + if e := coll.Insert(r); e != nil { + return e + } + } + return nil + }) + +``` + +## Dependencies + +- [globalsign/mgo](https://github.com/globalsign/mgo) - mgo mongo driver +- [stretchr/testify/](https://github.com/stretchr/testify) - testing library (test-only dependency) + +## Testing + +`testing.go` helps to create test for real mongo (not mocks) + +- `mongo.MakeTestConnection` creates `mongo.Connection` for url defined in env `MONGO_TEST`. If not defined `mongodb://mongo:27017` used. By default it will use random connection with prefix `test_` in `test` DB. +- `mongo.RemoveTestCollection` - drops collection used by `MakeTestConnection` +- `mongo.RemoveTestCollections` - drops user-defined collections from `test` DB + diff --git a/backend/app/store/engine/mongo/connection.go b/backend/vendor/github.com/go-pkgz/mongo/connection.go similarity index 96% rename from backend/app/store/engine/mongo/connection.go rename to backend/vendor/github.com/go-pkgz/mongo/connection.go index 769c8dbc..2f5ef87f 100644 --- a/backend/app/store/engine/mongo/connection.go +++ b/backend/vendor/github.com/go-pkgz/mongo/connection.go @@ -6,7 +6,7 @@ import ( "github.com/globalsign/mgo" ) -// sessionFn is a function for all With calls, terminator not supported +// sessionFn is a function for all With*Collection calls type sessionFn func(coll *mgo.Collection) error // Connection allows to run request in separate session, closing automatically diff --git a/backend/app/store/engine/mongo/server.go b/backend/vendor/github.com/go-pkgz/mongo/server.go similarity index 85% rename from backend/app/store/engine/mongo/server.go rename to backend/vendor/github.com/go-pkgz/mongo/server.go index e3a95bb5..31d7c0c0 100644 --- a/backend/app/store/engine/mongo/server.go +++ b/backend/vendor/github.com/go-pkgz/mongo/server.go @@ -1,9 +1,10 @@ // Package mongo wraps mgo to provide easier way to construct mongo server (with auth). -// Connection provides With* func warapers to run query with session copy +// Connection provides With* func wrappers to run query with session copy package mongo import ( "crypto/tls" + "errors" "fmt" "log" "net" @@ -12,7 +13,6 @@ import ( "time" "github.com/globalsign/mgo" - "github.com/pkg/errors" ) // Server represents mongo instance and provides session accessor @@ -24,9 +24,9 @@ type Server struct { // ServerParams optional set of parameters type ServerParams struct { + ConsistencyMode mgo.Mode Delay int // initial delay to give mongo server some time to start, in case if mongo part of the same compose Debug bool // turn on mgo debug mode - ConsistencyMode mgo.Mode SSL bool } @@ -35,14 +35,13 @@ type ServerParams struct { func NewServerWithURL(url string, timeout time.Duration) (res *Server, err error) { dial, params, err := parseURL(url, timeout) if err != nil { - return nil, errors.Wrapf(err, "failed to create mongo server from url %s", url) + return nil, fmt.Errorf("failed to create mongo server from url %s, %s", url, err) } return NewServer(dial, params) } // NewServer doing auth if passwd != "" and can delay to make sure local mongo is up func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error) { - log.Printf("[INFO] make new mongo server %v with %+v", dial.Addrs, params) result := Server{dial: dial, params: params} if params.Debug { @@ -71,9 +70,7 @@ func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error) session, err := mgo.DialWithInfo(&dial) if err != nil { - err = fmt.Errorf("can't connect to mongo, %v", err) - log.Printf("[ERROR] %v", err) - return nil, err + return nil, fmt.Errorf("can't connect to mongo, %v", err) } session.SetMode(params.ConsistencyMode, true) session.SetSyncTimeout(30 * time.Second) @@ -83,8 +80,7 @@ func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error) creds := &mgo.Credential{Username: dial.Username, Password: dial.Password, Source: dial.Source} log.Printf("[DEBUG] login to mongo, user=%s, db=%s", creds.Username, creds.Source) if err = session.Login(creds); err != nil { - log.Printf("[ERROR] can't login to mongo, %v", err) - return nil, err + return nil, fmt.Errorf("can't login to mongo, %v", err) } } @@ -114,7 +110,7 @@ func parseURL(mongoURL string, connectTimeout time.Duration) (mgo.DialInfo, Serv dial, err := mgo.ParseURL(mongoURL) if err != nil { - return mgo.DialInfo{}, ServerParams{}, errors.Wrapf(err, "failed to pars mongo url %s", mongoURL) + return mgo.DialInfo{}, ServerParams{}, fmt.Errorf("failed to pars mongo url %s, %s", mongoURL, err) } dial.Timeout = connectTimeout return *dial, params, nil diff --git a/backend/app/store/engine/mongo/testing.go b/backend/vendor/github.com/go-pkgz/mongo/testing.go similarity index 81% rename from backend/app/store/engine/mongo/testing.go rename to backend/vendor/github.com/go-pkgz/mongo/testing.go index 1c743913..d2cce4a9 100644 --- a/backend/app/store/engine/mongo/testing.go +++ b/backend/vendor/github.com/go-pkgz/mongo/testing.go @@ -9,7 +9,7 @@ import ( "time" "github.com/globalsign/mgo" - "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,24 +17,15 @@ import ( var conn *Connection var once sync.Once -// MakeTestConnection connects to MONGO_REMARK_TEST url or "mongo" host (in no env) and returns new connection. +// MakeTestConnection connects to MONGO_TEST url or "mongo" host (in no env) and returns new connection. // collection name randomized on each call func MakeTestConnection(t *testing.T) (*Connection, error) { - mongoURL := os.Getenv("MONGO_REMARK_TEST") - if mongoURL == "" { - t.Skip("no MONGO_REMARK_TEST in env") - return nil, errors.New("skip") - } - if mongoURL == "skip" { - t.Skip("skip mongo test") - return nil, errors.New("skip") - } - + mongoURL := getMongoURL(t) once.Do(func() { log.Print("[DEBUG] connect to mongo test instance") srv, err := NewServerWithURL(mongoURL, 10*time.Second) assert.Nil(t, err, "failed to dial") - collName := fmt.Sprintf("remark42_test_%d", time.Now().Nanosecond()) + collName := fmt.Sprintf("test_%d", time.Now().Nanosecond()) conn = NewConnection(srv, "test", collName) }) RemoveTestCollection(t, conn) @@ -71,3 +62,15 @@ func RemoveTestCollections(t *testing.T, c *Connection, collections ...string) { } } + +func getMongoURL(t *testing.T) string { + mongoURL := os.Getenv("MONGO_TEST") + if mongoURL == "" { + mongoURL = "mongodb://mongo:27017" + t.Logf("no MONGO_TEST in env, defaulted to %s", mongoURL) + } + if mongoURL == "skip" { + t.Skip("skip mongo test") + } + return mongoURL +} diff --git a/backend/app/store/engine/mongo/writer.go b/backend/vendor/github.com/go-pkgz/mongo/writer.go similarity index 91% rename from backend/app/store/engine/mongo/writer.go rename to backend/vendor/github.com/go-pkgz/mongo/writer.go index 74e9eb50..722044db 100644 --- a/backend/app/store/engine/mongo/writer.go +++ b/backend/vendor/github.com/go-pkgz/mongo/writer.go @@ -2,12 +2,12 @@ package mongo import ( "context" + "fmt" "log" "sync" "time" "github.com/globalsign/mgo" - "github.com/pkg/errors" ) // BufferedWriter defines interface for writes and flush @@ -76,7 +76,6 @@ func (bw *BufferedWriterMgo) WithAutoFlush(duration time.Duration) *BufferedWrit } } case <-bw.ctx.Done(): - log.Printf("[DEBUG] mongo writer flusher terminated") return } } @@ -92,9 +91,10 @@ func (bw *BufferedWriterMgo) Write(rec interface{}) error { bw.lastWriteTime = time.Now() bw.buffer = append(bw.buffer, rec) if len(bw.buffer) >= bw.bufferSize { - err := bw.writeBuffer() + if err := bw.writeBuffer(); err != nil { + return fmt.Errorf("failed to write to %s, %s", bw.connection, err) + } bw.buffer = bw.buffer[0:0] - return errors.Wrapf(err, "failed to write to %s", bw.connection) } return nil }) @@ -102,11 +102,15 @@ func (bw *BufferedWriterMgo) Write(rec interface{}) error { // Flush writes everything left in buffer to mongo func (bw *BufferedWriterMgo) Flush() error { - return bw.synced(func() error { + err := bw.synced(func() error { err := bw.writeBuffer() bw.buffer = bw.buffer[0:0] - return errors.Wrapf(err, "failed to flush to %s", bw.connection) + return err }) + if err != nil { + return fmt.Errorf("failed to flush to %s, %s", bw.connection, err) + } + return nil } // Close flushes all in-fly records and terminates background auto-flusher @@ -116,7 +120,6 @@ func (bw *BufferedWriterMgo) Close() (err error) { if bw.flushDuration > 0 { bw.cancel() <-bw.ctx.Done() - log.Printf("[DEBUG] mongo buffered writer closed") } return err }) diff --git a/compose-dev-backend.yml b/compose-dev-backend.yml index 7f96bc2a..8af2392b 100644 --- a/compose-dev-backend.yml +++ b/compose-dev-backend.yml @@ -18,7 +18,7 @@ services: dockerfile: Dockerfile args: - SKIP_FRONTEND_TEST=true - - MONGO_REMARK_TEST=skip # disable mongo tests on build by default. To allow remove =skip part and see above + - MONGO_TEST=skip # disable mongo tests on build by default. To allow remove =skip part and see above image: umputun/remark42:dev container_name: "remark42-dev" From 3520de768db4a2008cf7b910ff5c730d06126bf4 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 24 Jul 2018 18:43:53 -0400 Subject: [PATCH 30/93] feature/mongo cache (#180) * add siteID to cache Get * indirect option setters * add mongo cache with tests, add Key and Flusher * lint: minor warns * workaround for cache parallel test * repeater in mongo cache * missing repeater vendor * fix nop cache * add cache mongo benchmark * wired mongo cache, single opts group for mongo * disable goconst * stop cache repeated on not found error * use local mongo for tests in travis --- .travis.yml | 7 +- Dockerfile | 2 +- backend/Gopkg.lock | 11 +- backend/app/main.go | 54 ++-- backend/app/rest/api/admin.go | 14 +- backend/app/rest/api/migrator.go | 2 +- backend/app/rest/api/migrator_test.go | 3 +- backend/app/rest/api/rest_private.go | 8 +- backend/app/rest/api/rest_public.go | 23 +- backend/app/rest/api/rest_test.go | 11 +- backend/app/rest/api/rss.go | 9 +- backend/app/rest/cache/cache.go | 89 ++++++- backend/app/rest/cache/cache_test.go | 19 +- backend/app/rest/cache/memory.go | 48 +++- backend/app/rest/cache/memory_test.go | 78 +++--- backend/app/rest/cache/mongo.go | 188 ++++++++++++++ backend/app/rest/cache/mongo_test.go | 244 ++++++++++++++++++ backend/app/rest/cache/options.go | 33 +-- backend/remark.rest | 3 - .../github.com/go-pkgz/repeater/.gitignore | 12 + .../github.com/go-pkgz/repeater/.travis.yml | 18 ++ .../github.com/go-pkgz/repeater/LICENSE | 21 ++ .../github.com/go-pkgz/repeater/README.md | 36 +++ .../github.com/go-pkgz/repeater/repeater.go | 60 +++++ .../go-pkgz/repeater/strategy/backoff.go | 55 ++++ .../go-pkgz/repeater/strategy/fixed.go | 41 +++ .../go-pkgz/repeater/strategy/strategy.go | 28 ++ 27 files changed, 967 insertions(+), 150 deletions(-) create mode 100644 backend/app/rest/cache/mongo.go create mode 100644 backend/app/rest/cache/mongo_test.go create mode 100644 backend/vendor/github.com/go-pkgz/repeater/.gitignore create mode 100644 backend/vendor/github.com/go-pkgz/repeater/.travis.yml create mode 100644 backend/vendor/github.com/go-pkgz/repeater/LICENSE create mode 100644 backend/vendor/github.com/go-pkgz/repeater/README.md create mode 100644 backend/vendor/github.com/go-pkgz/repeater/repeater.go create mode 100644 backend/vendor/github.com/go-pkgz/repeater/strategy/backoff.go create mode 100644 backend/vendor/github.com/go-pkgz/repeater/strategy/fixed.go create mode 100644 backend/vendor/github.com/go-pkgz/repeater/strategy/strategy.go diff --git a/.travis.yml b/.travis.yml index 1c64345a..01016812 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ install: - docker-compose --version script: + - docker run -d --name=mongo mongo:3.6 && sleep 3 + - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo) + - echo "running mongo on $MONGO_TEST" - docker build --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN --build-arg CI=$CI @@ -16,5 +19,7 @@ script: --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG --build-arg TRAVIS_TAG=$TRAVIS_TAG - --build-arg MONGO_TEST=$MONGO_REMARK_TEST + --build-arg MONGO_TEST=$MONGO_TEST . + - docker rm -f mongo + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9f039463..2f4a1a24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \ if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \ - --enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \ + --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \ --enable=deadcode --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./... ; \ else echo "skip backend linters" ; fi diff --git a/backend/Gopkg.lock b/backend/Gopkg.lock index e512a56e..6c9c2cf2 100644 --- a/backend/Gopkg.lock +++ b/backend/Gopkg.lock @@ -87,6 +87,15 @@ revision = "9a09a574c336c6ae2338a65bbebed2baab2a713c" version = "v1.0.0" +[[projects]] + name = "github.com/go-pkgz/repeater" + packages = [ + ".", + "strategy" + ] + revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b" + version = "v1.0.0" + [[projects]] name = "github.com/golang/protobuf" packages = ["proto"] @@ -256,6 +265,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "e933250b2582d1ff7d2b165c4a8a934762677a2ebaaea743fb3c485868c6d793" + inputs-digest = "da9242a76135f9ee04287bf437ffa42708fc36ec54f39c74127f41a3473f24ac" solver-name = "gps-cdcl" solver-version = 1 diff --git a/backend/app/main.go b/backend/app/main.go index 343a4dba..0e1d1aa3 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -28,7 +28,6 @@ import ( ) // Opts with command line flags and env -// nolint:maligned type Opts struct { SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"` RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` @@ -36,6 +35,7 @@ type Opts struct { Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` + Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` @@ -79,7 +79,6 @@ type StoreGroup struct { Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` - Mongo MongoOpts `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` } // AvatarGroup defines options group for avatar params @@ -88,13 +87,12 @@ type AvatarGroup struct { FS struct { Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` } `group:"fs" namespace:"fs" env-namespace:"FS"` - Mongo MongoOpts `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` - RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` + RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` } // CacheGroup defines options group for cache params type CacheGroup struct { - Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"redis" default:"mem"` + Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"` Max struct { Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` @@ -102,8 +100,8 @@ type CacheGroup struct { } `group:"max" namespace:"max" env-namespace:"MAX"` } -// MongoOpts holds all mongo params -type MongoOpts struct { +// MongoGroup holds all mongo params, used by store, avatar and cache +type MongoGroup struct { URL string `long:"url" env:"URL" description:"mongo url"` DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` } @@ -163,7 +161,7 @@ func New(opts Opts) (*Application, error) { return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL) } - storeEngine, err := makeDataStore(opts.Store, opts.Sites) + storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites) if err != nil { return nil, err } @@ -176,8 +174,7 @@ func New(opts Opts) (*Application, error) { Admins: opts.Admins, } - loadingCache, err := cache.NewMemoryCache(cache.MaxCacheSize(opts.Cache.Max.Size), cache.MaxValSize(opts.Cache.Max.Value), - cache.MaxKeys(opts.Cache.Max.Items)) + loadingCache, err := makeCache(opts.Cache, opts.Mongo) if err != nil { return nil, err } @@ -186,7 +183,7 @@ func New(opts Opts) (*Application, error) { jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) - avatarStore, err := makeAvatarStore(opts.Avatar) + avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) if err != nil { return nil, errors.Wrap(err, "failed to make avatar store") } @@ -296,7 +293,7 @@ func (a *Application) activateBackup(ctx context.Context) { } // makeDataStore creates store for all sites -func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interface, err error) { +func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) { switch group.Type { case "bolt": if err = makeDirs(group.Bolt.Path); err != nil { @@ -308,11 +305,11 @@ func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interfac } result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...) case "mongo": - mgServer, e := makeMongo(group.Mongo) + mgServer, e := makeMongo(mg) if e != nil { return result, errors.Wrap(e, "failed to create mongo server") } - conn := mongo.NewConnection(mgServer, group.Mongo.DB, "") + conn := mongo.NewConnection(mgServer, mg.DB, "") result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) default: return nil, errors.Errorf("unsupported store type %s", group.Type) @@ -320,7 +317,7 @@ func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interfac return result, errors.Wrap(err, "can't initialize data store") } -func makeAvatarStore(group AvatarGroup) (avatar.Store, error) { +func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { switch group.Type { case "fs": if err := makeDirs(group.FS.Path); err != nil { @@ -328,16 +325,33 @@ func makeAvatarStore(group AvatarGroup) (avatar.Store, error) { } return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil case "mongo": - mgServer, err := makeMongo(group.Mongo) + mgServer, err := makeMongo(mg) if err != nil { return nil, errors.Wrap(err, "failed to create mongo server") } - conn := mongo.NewConnection(mgServer, group.Mongo.DB, "") + conn := mongo.NewConnection(mgServer, mg.DB, "") return avatar.NewGridFS(conn, group.RszLmt), nil } return nil, errors.Errorf("unsupported avatar store type %s", group.Type) } +func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { + switch group.Type { + case "mem": + return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), + cache.MaxKeys(group.Max.Items)) + case "mongo": + mgServer, err := makeMongo(mg) + if err != nil { + return nil, errors.Wrap(err, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, mg.DB, "cache") + return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), + cache.MaxKeys(group.Max.Items)) + } + return nil, errors.Errorf("unsupported cache type %s", group.Type) +} + // mkdir -p for all dirs func makeDirs(dirs ...string) error { @@ -367,11 +381,11 @@ func makeDirs(dirs ...string) error { return nil } -func makeMongo(mopts MongoOpts) (result *mongo.Server, err error) { - if mopts.URL == "" { +func makeMongo(mg MongoGroup) (result *mongo.Server, err error) { + if mg.URL == "" { return nil, errors.New("no mongo URL provided") } - return mongo.NewServerWithURL(mopts.URL, 10*time.Second) + return mongo.NewServerWithURL(mg.URL, 10*time.Second) } func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider { diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 04baee07..c1fd52e8 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -57,7 +57,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment") return } - a.cache.Flush(locator.SiteID, locator.URL) + a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last")) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"id": id, "locator": locator}) } @@ -73,7 +73,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user") return } - a.cache.Flush(siteID, userID) + a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID)) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID}) } @@ -118,7 +118,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user") return } - a.cache.Flush(claims.SiteID, claims.User.ID) + a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, "last")) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID}) } @@ -140,7 +140,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status") return } - a.cache.Flush(siteID, userID) + a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID)) render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus}) } @@ -177,7 +177,7 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status") return } - a.cache.Flush(locator.SiteID) + a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID)) render.JSON(w, r, JSON{"locator": locator, "read-only": roStatus}) } @@ -191,7 +191,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status") return } - a.cache.Flush(siteID, userID) + a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID)) render.JSON(w, r, JSON{"user": userID, "verified": verifyStatus}) } @@ -206,7 +206,7 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status") return } - a.cache.Flush(locator.URL) + a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL)) render.JSON(w, r, JSON{"id": commentID, "locator": locator, "pin": pinStatus}) } diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 8e17c9ac..c8bba0da 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -98,7 +98,7 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") return } - m.Cache.Flush(siteID) + m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID)) render.Status(r, http.StatusCreated) render.JSON(w, r, JSON{"status": "ok", "size": size}) diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index d141dae1..98a34908 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/migrator" + "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) @@ -111,7 +112,7 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) { DisqusImporter: &migrator.Disqus{DataStore: dataStore}, NativeImporter: &migrator.Remark{DataStore: dataStore}, NativeExported: &migrator.Remark{DataStore: dataStore}, - Cache: &mockCache{}, + Cache: &cache.Nop{}, SecretKey: "123456", } diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 5435f570..62a2163a 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -18,6 +18,7 @@ import ( "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/auth" + "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/service" ) @@ -70,7 +71,8 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment") return } - s.Cache.Flush(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID) + s.Cache.Flush(cache.Flusher(comment.Locator.SiteID). + Scopes(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)) render.Status(r, http.StatusCreated) render.JSON(w, r, &finalComment) @@ -121,7 +123,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - s.Cache.Flush(locator.URL, "last", user.ID) + s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last", user.ID)) render.JSON(w, r, res) } @@ -155,7 +157,7 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment") return } - s.Cache.Flush(locator.URL, comment.User.ID) + s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID)) render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score}) } diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index b4fd0eda..bf397ab0 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -27,7 +27,8 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) { } log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format")) - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) { + key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL) + data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Find(locator, sort) if e != nil { return nil, e @@ -90,7 +91,8 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) { func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) { + key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL) + data, err := s.Cache.Get(key, func() ([]byte, error) { info, e := s.DataService.Info(locator, s.ReadOnlyAge) if e != nil { return nil, e @@ -116,7 +118,8 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) { limit = 0 } - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), "last", siteID), func() ([]byte, error) { + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes("last") + data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, limit) if e != nil { return nil, e @@ -179,7 +182,8 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) { log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID) - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), userID, siteID), func() ([]byte, error) { + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(userID, siteID) + data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.User(siteID, userID, limit, 0) if e != nil { return nil, e @@ -259,15 +263,15 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) { } // key could be long for multiple posts, make it sha1 - key := cache.URLKey(r) + strings.Join(posts, ",") + k := cache.URLKey(r) + strings.Join(posts, ",") hasher := sha1.New() - if _, err := hasher.Write([]byte(key)); err != nil { + if _, err := hasher.Write([]byte(k)); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls") return } sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil)) - - data, err := s.Cache.Get(cache.Key(sha, siteID), func() ([]byte, error) { + key := cache.NewKey(siteID).ID(sha).Scopes(siteID) + data, err := s.Cache.Get(key, func() ([]byte, error) { counts, e := s.DataService.Counts(siteID, posts) if e != nil { return nil, e @@ -295,7 +299,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) { skip = v } - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), func() ([]byte, error) { + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID) + data, err := s.Cache.Get(key, func() ([]byte, error) { posts, e := s.DataService.List(siteID, limit, skip) if e != nil { return nil, e diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 6dd65230..936e8df2 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -17,6 +17,7 @@ import ( "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/auth" + "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/avatar" @@ -90,7 +91,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { JWTService: auth.NewJWT("12345", false, time.Minute, time.Hour), }, Exporter: &migrator.Remark{DataStore: dataStore}, - Cache: &mockCache{}, + Cache: &cache.Nop{}, WebRoot: "/tmp", RemarkURL: "https://demo.remark42.com", AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"}, @@ -163,11 +164,3 @@ func cleanup(ts *httptest.Server, srv *Rest) { os.Remove(testDb) os.Remove(testHTML) } - -type mockCache struct{} - -func (mc *mockCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) { - return fn() -} - -func (mc *mockCache) Flush(scopes ...string) {} diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go index 12245848..39ef0fb6 100644 --- a/backend/app/rest/api/rss.go +++ b/backend/app/rest/api/rss.go @@ -35,7 +35,8 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} log.Printf("[DEBUG] get rss for post %+v", locator) - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) { + key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL) + data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Find(locator, "-time") if e != nil { return nil, e @@ -66,7 +67,8 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss for site %s", siteID) - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID, "last"), func() ([]byte, error) { + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last") + data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, maxRssItems) if e != nil { return nil, e @@ -98,7 +100,8 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID) - data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID, "last"), func() (res []byte, e error) { + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last") + data, err := s.Cache.Get(key, func() (res []byte, e error) { comments, e := s.DataService.Last(siteID, maxLastCommentsReply) if e != nil { return nil, errors.Wrap(e, "can't get last comments") diff --git a/backend/app/rest/cache/cache.go b/backend/app/rest/cache/cache.go index 10badd95..b0c4d62c 100644 --- a/backend/app/rest/cache/cache.go +++ b/backend/app/rest/cache/cache.go @@ -5,32 +5,88 @@ import ( "strings" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/rest" ) // LoadingCache defines interface for caching type LoadingCache interface { - Get(key string, fn func() ([]byte, error)) (data []byte, err error) - Flush(scopes ...string) + Get(key Key, fn func() ([]byte, error)) (data []byte, err error) + Flush(req FlusherRequest) } -// Key makes full key from primary key and scopes -func Key(key string, scopes ...string) string { - return strings.Join(scopes, "$$") + "@@" + key +type cacheWithOpts interface { + LoadingCache + setMaxValSize(max int) error + setMaxKeys(max int) error + setMaxCacheSize(max int64) error + setPostFlushFn(postFlushFn func()) error +} + +// Key for cache +type Key struct { + id string + siteID string + scopes []string +} + +// NewKey makes keys for site +func NewKey(site string) Key { + res := Key{siteID: site} + return res +} + +// ID sets key id +func (k Key) ID(id string) Key { + k.id = id + return k +} + +// Scopes of the key +func (k Key) Scopes(scopes ...string) Key { + k.scopes = scopes + return k +} + +// Merge makes full string key from primary key and scopes +func (k Key) Merge() string { + return strings.Join(k.scopes, "$$") + "@@" + k.id + "@@" + k.siteID } // ParseKey gets compound key created by Key func and split it to the actual key and scopes -func ParseKey(fullKey string) (key string, scopes []string, err error) { +func ParseKey(fullKey string) (Key, error) { elems := strings.Split(fullKey, "@@") - if len(elems) != 2 { - return "", nil, errors.Errorf("can't parse cache key %s", key) + if len(elems) != 3 { + return Key{}, errors.Errorf("can't parse cache key %s", fullKey) } - scopes = strings.Split(elems[0], "$$") + scopes := strings.Split(elems[0], "$$") if len(scopes) == 1 && scopes[0] == "" { scopes = []string{} } - key = elems[1] - return key, scopes, nil + key := Key{ + scopes: scopes, + id: elems[1], + siteID: elems[2], + } + return key, nil +} + +// FlusherRequest used as input for cache.Flush +type FlusherRequest struct { + siteID string + scopes []string +} + +// Flusher makes new FlusherRequest with empty scopes +func Flusher(siteID string) FlusherRequest { + res := FlusherRequest{siteID: siteID} + return res +} + +// Scopes adds scopes to FlusherRequest +func (f FlusherRequest) Scopes(scopes ...string) FlusherRequest { + f.scopes = scopes + return f } // URLKey gets url from request to use it as cache key @@ -43,3 +99,14 @@ func URLKey(r *http.Request) string { } return key } + +// Nop does nothing for caching, passing fn call only +type Nop struct{} + +// Get calls fn, no actual caching +func (n *Nop) Get(key Key, fn func() ([]byte, error)) (data []byte, err error) { + return fn() +} + +// Flush does nothing for NoopCache +func (n *Nop) Flush(req FlusherRequest) {} diff --git a/backend/app/rest/cache/cache_test.go b/backend/app/rest/cache/cache_test.go index 31d934c5..8b0e0645 100644 --- a/backend/app/rest/cache/cache_test.go +++ b/backend/app/rest/cache/cache_test.go @@ -15,24 +15,25 @@ func TestCache_Keys(t *testing.T) { scopes []string full string }{ - {"key1", []string{"s1"}, "s1@@key1"}, - {"key2", []string{"s11", "s2"}, "s11$$s2@@key2"}, - {"key3", []string{}, "@@key3"}, + {"key1", []string{"s1"}, "s1@@key1@@site"}, + {"key2", []string{"s11", "s2"}, "s11$$s2@@key2@@site"}, + {"key3", []string{}, "@@key3@@site"}, } for n, tt := range tbl { - full := Key(tt.key, tt.scopes...) + k := NewKey("site").ID(tt.key).Scopes(tt.scopes...) + full := k.Merge() assert.Equal(t, tt.full, full, "making key, #%d", n) - k, s, e := ParseKey(full) + k, e := ParseKey(full) assert.Nil(t, e) - assert.Equal(t, tt.scopes, s) - assert.Equal(t, tt.key, k) + assert.Equal(t, tt.scopes, k.scopes) + assert.Equal(t, tt.key, k.id) } - _, _, err := ParseKey("abc") + _, err := ParseKey("abc") assert.Error(t, err) - _, _, err = ParseKey("") + _, err = ParseKey("") assert.Error(t, err) } diff --git a/backend/app/rest/cache/memory.go b/backend/app/rest/cache/memory.go index 1426f999..a40153b0 100644 --- a/backend/app/rest/cache/memory.go +++ b/backend/app/rest/cache/memory.go @@ -20,6 +20,8 @@ type memoryCache struct { // NewMemoryCache makes memoryCache implementation func NewMemoryCache(options ...Option) (LoadingCache, error) { + log.Print("[INFO] make memory cache") + res := memoryCache{ postFlushFn: func() {}, maxKeys: 1000, @@ -48,8 +50,9 @@ func NewMemoryCache(options ...Option) (LoadingCache, error) { } // Get is loading cache method to get value by key or load via fn if not found -func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) { - if b, ok := m.bytesCache.Get(key); ok { +func (m *memoryCache) Get(key Key, fn func() ([]byte, error)) (data []byte, err error) { + mkey := key.Merge() + if b, ok := m.bytesCache.Get(mkey); ok { return b.([]byte), nil } @@ -57,7 +60,7 @@ func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, e return data, err } if m.allowed(data) { - m.bytesCache.Add(key, data) + m.bytesCache.Add(mkey, data) atomic.AddInt64(&m.currentSize, int64(len(data))) if m.maxCacheSize > 0 && atomic.LoadInt64(&m.currentSize) > m.maxCacheSize { @@ -70,9 +73,9 @@ func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, e } // Flush clears cache and calls postFlushFn async -func (m *memoryCache) Flush(scopes ...string) { +func (m *memoryCache) Flush(req FlusherRequest) { - if len(scopes) == 0 { + if len(req.scopes) == 0 { m.bytesCache.Purge() go m.postFlushFn() return @@ -80,12 +83,12 @@ func (m *memoryCache) Flush(scopes ...string) { // check if fullKey has matching scopes inScope := func(fullKey string) bool { - _, keyScopes, err := ParseKey(fullKey) + key, err := ParseKey(fullKey) if err != nil { return false } - for _, s := range scopes { - for _, ks := range keyScopes { + for _, s := range req.scopes { + for _, ks := range key.scopes { if ks == s { return true } @@ -112,3 +115,32 @@ func (m *memoryCache) allowed(data []byte) bool { } return true } + +func (m *memoryCache) setMaxValSize(max int) error { + m.maxValueSize = max + if max <= 0 { + return errors.Errorf("negative size for MaxValSize, %d", max) + } + return nil +} + +func (m *memoryCache) setMaxKeys(max int) error { + m.maxKeys = max + if max <= 0 { + return errors.Errorf("negative size for MaxKeys, %d", max) + } + return nil +} + +func (m *memoryCache) setMaxCacheSize(max int64) error { + m.maxCacheSize = max + if max <= 0 { + return errors.Errorf("negative size or MaxCacheSize, %d", max) + } + return nil +} + +func (m *memoryCache) setPostFlushFn(postFlushFn func()) error { + m.postFlushFn = postFlushFn + return nil +} diff --git a/backend/app/rest/cache/memory_test.go b/backend/app/rest/cache/memory_test.go index eb03bad5..35fe6b6a 100644 --- a/backend/app/rest/cache/memory_test.go +++ b/backend/app/rest/cache/memory_test.go @@ -17,7 +17,7 @@ func TestMemoryCache_Get(t *testing.T) { var postFnCall, coldCalls int32 lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) })) require.Nil(t, err) - res, err := lc.Get("key", func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { atomic.AddInt32(&coldCalls, 1) return []byte("result"), nil }) @@ -26,7 +26,7 @@ func TestMemoryCache_Get(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - res, err = lc.Get("key", func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { atomic.AddInt32(&coldCalls, 1) return []byte("result"), nil }) @@ -35,11 +35,11 @@ func TestMemoryCache_Get(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) - lc.Flush() + lc.Flush(Flusher("site")) time.Sleep(100 * time.Millisecond) // let postFn to do its thing assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall)) - _, err = lc.Get("key", func() ([]byte, error) { + _, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { return nil, errors.New("err") }) assert.NotNil(t, err) @@ -53,7 +53,7 @@ func TestMemoryCache_MaxKeys(t *testing.T) { // put 5 keys to cache for i := 0; i < 5; i++ { - res, e := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) { + res, e := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) { atomic.AddInt32(&coldCalls, 1) return []byte(fmt.Sprintf("result-%d", i)), nil }) @@ -64,14 +64,14 @@ func TestMemoryCache_MaxKeys(t *testing.T) { } // check if really cached - res, err := lc.Get("key-3", func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID("key-3"), func() ([]byte, error) { return []byte("result-blah"), nil }) assert.Nil(t, err) assert.Equal(t, "result-3", string(res), "should be cached") // try to cache after maxKeys reached - res, err = lc.Get("key-X", func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key-X"), func() ([]byte, error) { return []byte("result-X"), nil }) assert.Nil(t, err) @@ -80,13 +80,13 @@ func TestMemoryCache_MaxKeys(t *testing.T) { assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len()) // put to cache and make sure it cached - res, err = lc.Get("key-Z", func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { return []byte("result-Z"), nil }) assert.Nil(t, err) assert.Equal(t, "result-Z", string(res)) - res, err = lc.Get("key-Z", func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { return []byte("result-Zzzz"), nil }) assert.Nil(t, err) @@ -98,26 +98,26 @@ func TestMemoryCache_MaxValueSize(t *testing.T) { lc, err := NewMemoryCache(MaxKeys(5), MaxValSize(10)) require.Nil(t, err) // put good size value to cache and make sure it cached - res, err := lc.Get("key-Z", func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { return []byte("result-Z"), nil }) assert.Nil(t, err) assert.Equal(t, "result-Z", string(res)) - res, err = lc.Get("key-Z", func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { return []byte("result-Zzzz"), nil }) assert.Nil(t, err) assert.Equal(t, "result-Z", string(res), "got cached value") // put too big value to cache and make sure it is not cached - res, err = lc.Get("key-Big", func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { return []byte("1234567890"), nil }) assert.Nil(t, err) assert.Equal(t, "1234567890", string(res)) - res, err = lc.Get("key-Big", func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { return []byte("result-big"), nil }) assert.Nil(t, err) @@ -129,21 +129,21 @@ func TestMemoryCache_MaxCacheSize(t *testing.T) { require.Nil(t, err) // put good size value to cache and make sure it cached - res, err := lc.Get("key-Z", func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { return []byte("result-Z"), nil }) assert.Nil(t, err) assert.Equal(t, "result-Z", string(res)) assert.Equal(t, int64(8), lc.(*memoryCache).currentSize) - _, err = lc.Get("key-Z2", func() ([]byte, error) { + _, err = lc.Get(NewKey("site").ID("key-Z2"), func() ([]byte, error) { return []byte("result-Z"), nil }) assert.Nil(t, err) assert.Equal(t, int64(16), lc.(*memoryCache).currentSize) // this will cause removal - _, err = lc.Get("key-Z3", func() ([]byte, error) { + _, err = lc.Get(NewKey("site").ID("key-Z3"), func() ([]byte, error) { return []byte("result-Z"), nil }) assert.Nil(t, err) @@ -163,7 +163,7 @@ func TestMemoryCache_MaxCacheSizeParallel(t *testing.T) { go func() { time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond) defer wg.Done() - res, err := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) { return []byte(fmt.Sprintf("result-%d", i)), nil }) require.Nil(t, err) @@ -182,7 +182,7 @@ func TestMemoryCache_Parallel(t *testing.T) { lc, err := NewMemoryCache() require.Nil(t, err) - res, err := lc.Get("key", func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { return []byte("value"), nil }) assert.Nil(t, err) @@ -194,7 +194,7 @@ func TestMemoryCache_Parallel(t *testing.T) { i := i go func() { defer wg.Done() - res, err := lc.Get("key", func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { atomic.AddInt32(&coldCalls, 1) return []byte(fmt.Sprintf("result-%d", i)), nil }) @@ -210,28 +210,28 @@ func TestMemoryCache_Scopes(t *testing.T) { lc, err := NewMemoryCache() require.Nil(t, err) - res, err := lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) { + res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { return []byte("value"), nil }) assert.Nil(t, err) assert.Equal(t, "value", string(res)) - res, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { return []byte("value2"), nil }) assert.Nil(t, err) assert.Equal(t, "value2", string(res)) assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len()) - lc.Flush("s1") + lc.Flush(Flusher("site").Scopes("s1")) assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) - _, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) { + _, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { assert.Fail(t, "should stay") return nil, nil }) assert.Nil(t, err) - res, err = lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) { + res, err = lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { return []byte("value-upd"), nil }) assert.Nil(t, err) @@ -242,23 +242,23 @@ func TestMemoryCache_Flush(t *testing.T) { lc, err := NewMemoryCache() require.Nil(t, err) - addToCache := func(key string, scopes ...string) { - res, err := lc.Get(key, func() ([]byte, error) { - return []byte("value" + key), nil + addToCache := func(id string, scopes ...string) { + res, err := lc.Get(NewKey("site").ID(id).Scopes(scopes...), func() ([]byte, error) { + return []byte("value" + id), nil }) require.Nil(t, err) - require.Equal(t, "value"+key, string(res)) + require.Equal(t, "value"+id, string(res)) } init := func() { - lc.Flush() - addToCache(Key("key1", "s1", "s2")) - addToCache(Key("key2", "s1", "s2", "s3")) - addToCache(Key("key3", "s1", "s2", "s3")) - addToCache(Key("key4", "s2", "s3")) - addToCache(Key("key5", "s2")) - addToCache(Key("key6")) - addToCache(Key("key7", "s4", "s3")) + lc.Flush(Flusher("site")) + addToCache("key1", "s1", "s2") + addToCache("key2", "s1", "s2", "s3") + addToCache("key3", "s1", "s2", "s3") + addToCache("key4", "s2", "s3") + addToCache("key5", "s2") + addToCache("key6") + addToCache("key7", "s4", "s3") require.Equal(t, 7, lc.(*memoryCache).bytesCache.Len(), "cache init") } @@ -279,7 +279,7 @@ func TestMemoryCache_Flush(t *testing.T) { for i, tt := range tbl { init() - lc.Flush(tt.scopes...) + lc.Flush(Flusher("site").Scopes(tt.scopes...)) assert.Equal(t, tt.left, lc.(*memoryCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i) } } @@ -287,14 +287,14 @@ func TestMemoryCache_Flush(t *testing.T) { func TestMemoryCache_FlushFailed(t *testing.T) { lc, err := NewMemoryCache() require.Nil(t, err) - val, err := lc.Get("invalid-composite", func() ([]byte, error) { + val, err := lc.Get(NewKey("site").ID("invalid-composite"), func() ([]byte, error) { return []byte("value"), nil }) assert.Nil(t, err) assert.Equal(t, "value", string(val)) assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) - lc.Flush("invalid-composite") + lc.Flush(Flusher("site").Scopes("invalid-composite")) assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len()) } diff --git a/backend/app/rest/cache/mongo.go b/backend/app/rest/cache/mongo.go new file mode 100644 index 00000000..8f0c8e99 --- /dev/null +++ b/backend/app/rest/cache/mongo.go @@ -0,0 +1,188 @@ +package cache + +import ( + "log" + "time" + + "github.com/go-pkgz/repeater" + + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + "github.com/go-pkgz/mongo" + multierror "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" +) + +type mongoCache struct { + connection *mongo.Connection + postFlushFn func() + maxKeys int + maxValueSize int + maxCacheSize int64 +} + +const cacheCollection = "cache" + +type mongoDoc struct { + SiteID string `bson:"site"` + Key string `bson:"key"` + Scopes []string `bson:"scopes,omitempty"` + Data []byte `bson:"data"` +} + +// NewMongoCache makes mongoCache implementation +func NewMongoCache(connection *mongo.Connection, options ...Option) (LoadingCache, error) { + log.Printf("[INFO] make mongo cache with %s", connection) + res := &mongoCache{ + connection: connection, + postFlushFn: func() {}, + maxKeys: 1000, + maxValueSize: 0, + } + for _, opt := range options { + if err := opt(res); err != nil { + return nil, errors.Wrap(err, "failed to set cache option") + } + } + if err := res.prepare(); err != nil { + return nil, err + } + return res, nil +} + +// Get is loading cache method to get value by key or load via fn if not found +func (m *mongoCache) Get(key Key, fn func() ([]byte, error)) (data []byte, err error) { + + d := mongoDoc{} + + // repeat find from cache with small delay to avoid mgo random error + rep := repeater.NewDefault(5, 10*time.Millisecond) + mgErr := rep.Do(func() error { + return m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": key.siteID, "key": key.id}).One(&d) + }) + }, mgo.ErrNotFound) + if mgErr == nil { // cached result found + return d.Data, nil + } + + if data, err = fn(); err != nil { + return data, err + } + + if mgErr != mgo.ErrNotFound { // some other error in mgo query, don't try to update cache + log.Printf("[WARN] unexpected mgo error %+v", mgErr) + return data, err + } + + if !m.allowed(data) { + return data, nil + } + + d = mongoDoc{ + SiteID: key.siteID, + Key: key.id, + Data: data, + Scopes: key.scopes, + } + err = m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { + _, e := coll.Upsert(bson.M{"site": key.siteID, "key": key.id}, bson.M{"$set": d}) + return e + }) + if err != nil { + return nil, errors.Wrapf(err, "can't set cached value for %+v", key) + } + + if m.maxKeys > 0 { + err = m.cleanup(key.siteID) + } + + return data, errors.Wrap(err, "failed to cleanup cached records") +} + +func (m *mongoCache) cleanup(siteID string) (err error) { + ids := []struct { + ID bson.ObjectId `bson:"_id"` + }{} + + err = m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { + n, countErr := coll.Find(bson.M{"site": siteID}).Count() + if countErr != nil { + return countErr + } + if countErr == nil && n > m.maxKeys { + if findErr := coll.Find(bson.M{"site": siteID}).Sort("+id").Limit(n - m.maxKeys).All(&ids); findErr == nil { + bsonIDs := []bson.ObjectId{} + for _, id := range ids { + bsonIDs = append(bsonIDs, id.ID) + } + _, removalErr := coll.RemoveAll(bson.M{"_id": bson.M{"$in": bsonIDs}}) + return removalErr + } + } + return nil + }) + return err +} + +// Flush clears cache and calls postFlushFn async +func (m *mongoCache) Flush(req FlusherRequest) { + err := m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { + q := bson.M{"site": req.siteID} + if len(req.scopes) > 0 { + q["scopes"] = bson.M{"$in": req.scopes} + } + _, e := coll.RemoveAll(q) + return e + }) + + if err == nil && m.postFlushFn != nil { + m.postFlushFn() + } +} + +// prepare collections with all indexes +func (m *mongoCache) prepare() error { + errs := new(multierror.Error) + return m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error { + errs = multierror.Append(errs, coll.EnsureIndexKey("site", "key")) + errs = multierror.Append(errs, coll.EnsureIndexKey("site", "scopes")) + return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", cacheCollection) + }) +} + +func (m *mongoCache) allowed(data []byte) bool { + if m.maxValueSize > 0 && len(data) >= m.maxValueSize { + return false + } + return true +} + +func (m *mongoCache) setMaxValSize(max int) error { + m.maxValueSize = max + if max <= 0 { + return errors.Errorf("negative size for MaxValSize, %d", max) + } + return nil +} + +func (m *mongoCache) setMaxKeys(max int) error { + m.maxKeys = max + if max <= 0 { + return errors.Errorf("negative size for MaxKeys, %d", max) + } + return nil +} + +func (m *mongoCache) setMaxCacheSize(max int64) error { + m.maxCacheSize = max + if max <= 0 { + return errors.Errorf("negative size or MaxCacheSize, %d", max) + } + return nil +} + +func (m *mongoCache) setPostFlushFn(postFlushFn func()) error { + m.postFlushFn = postFlushFn + return nil +} diff --git a/backend/app/rest/cache/mongo_test.go b/backend/app/rest/cache/mongo_test.go new file mode 100644 index 00000000..be2a7e4c --- /dev/null +++ b/backend/app/rest/cache/mongo_test.go @@ -0,0 +1,244 @@ +package cache + +import ( + "fmt" + "log" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMongoCache_Get(t *testing.T) { + conn, err := mongo.MakeTestConnection(t) + assert.NoError(t, err) + defer mongo.RemoveTestCollections(t, conn, "cache") + + var postFnCall, coldCalls int32 + lc, err := NewMongoCache(conn, PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) })) + require.Nil(t, err) + res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte("result"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result", string(res)) + assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) + assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + + res, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte("result"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result", string(res)) + assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls)) + assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + + lc.Flush(Flusher("site")) + time.Sleep(100 * time.Millisecond) // let postFn to do its thing + assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall)) + + _, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) { + return nil, errors.New("err") + }) + assert.NotNil(t, err) +} + +func TestMongoCache_MaxKeys(t *testing.T) { + var postFnCall, coldCalls int32 + conn, err := mongo.MakeTestConnection(t) + assert.NoError(t, err) + defer mongo.RemoveTestCollections(t, conn, "cache") + + lc, err := NewMongoCache(conn, PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }), + MaxKeys(5), MaxValSize(10)) + require.Nil(t, err) + + // put 5 keys to cache + for i := 0; i < 5; i++ { + res, e := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte(fmt.Sprintf("result-%d", i)), nil + }) + assert.Nil(t, e) + assert.Equal(t, fmt.Sprintf("result-%d", i), string(res)) + assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls)) + assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall)) + } + + // check if really cached + res, err := lc.Get(NewKey("site").ID("key-3"), func() ([]byte, error) { + return []byte("result-blah"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-3", string(res), "should be cached") + + // try to cache after maxKeys reached + res, err = lc.Get(NewKey("site").ID("key-X"), func() ([]byte, error) { + return []byte("result-X"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-X", string(res)) + + conn.WithCustomCollection("cache", func(coll *mgo.Collection) error { + n, e := coll.Find(bson.M{"site": "site"}).Count() + require.NoError(t, e) + require.Equal(t, 5, n) + r := mongoDoc{} + require.NoError(t, coll.Find(bson.M{"site": "site"}).Sort("+_id").One(&r)) + assert.Equal(t, "key-1", r.Key) + return nil + }) + + // put to cache and make sure it cached + res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { + return []byte("result-Z"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res)) + + res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { + return []byte("result-Zzzz"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res), "got cached value") + + conn.WithCustomCollection("cache", func(coll *mgo.Collection) error { + n, e := coll.Find(bson.M{"site": "site"}).Count() + require.NoError(t, e) + require.Equal(t, 5, n) + r := mongoDoc{} + require.NoError(t, coll.Find(bson.M{"site": "site"}).Sort("+_id").One(&r)) + assert.Equal(t, "key-2", r.Key) + return nil + }) +} + +func TestMongoCache_Parallel(t *testing.T) { + var coldCalls int32 + conn, err := mongo.MakeTestConnection(t) + assert.NoError(t, err) + defer mongo.RemoveTestCollections(t, conn, "cache") + lc, err := NewMongoCache(conn) + require.Nil(t, err) + + res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { + return []byte("value"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value", string(res)) + wg := sync.WaitGroup{} + for i := 0; i < 100; i++ { + wg.Add(1) + i := i + go func() { + defer wg.Done() + r, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { + atomic.AddInt32(&coldCalls, 1) + return []byte(fmt.Sprintf("result-%d", i)), nil + }) + require.Nil(t, err) + v := string(r) + assert.Equal(t, "value", v, "th=%d", i) + }() + } + wg.Wait() + assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls)) +} + +func TestMongoCache_Flush(t *testing.T) { + conn, err := mongo.MakeTestConnection(t) + assert.NoError(t, err) + defer mongo.RemoveTestCollections(t, conn, "cache") + lc, err := NewMongoCache(conn) + require.Nil(t, err) + + addToCache := func(id string, scopes ...string) { + res, err := lc.Get(NewKey("site").ID(id).Scopes(scopes...), func() ([]byte, error) { + return []byte("value" + id), nil + }) + require.Nil(t, err) + require.Equal(t, "value"+id, string(res)) + } + + cacheSize := func() (count int) { + conn.WithCustomCollection("cache", func(coll *mgo.Collection) (e error) { + count, e = coll.Find(bson.M{"site": "site"}).Count() + require.NoError(t, e) + return e + }) + return count + } + + init := func() { + lc.Flush(Flusher("site")) + addToCache("key1", "s1", "s2") + addToCache("key2", "s1", "s2", "s3") + addToCache("key3", "s1", "s2", "s3") + addToCache("key4", "s2", "s3") + addToCache("key5", "s2") + addToCache("key6") + addToCache("key7", "s4", "s3") + require.Equal(t, 7, cacheSize(), "cache init") + } + + tbl := []struct { + scopes []string + left int + msg string + }{ + {[]string{}, 0, "full flush, no scopes"}, + {[]string{"s0"}, 7, "flush wrong scope"}, + {[]string{"s1"}, 4, "flush s1 scope"}, + {[]string{"s2", "s1"}, 2, "flush s2+s1 scope"}, + {[]string{"s1", "s2"}, 2, "flush s1+s2 scope"}, + {[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"}, + {[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"}, + {[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"}, + } + + for i, tt := range tbl { + init() + lc.Flush(Flusher("site").Scopes(tt.scopes...)) + assert.Equal(t, tt.left, cacheSize(), "keys size, %s #%d", tt.msg, i) + } +} + +func BenchmarkMongoCache(b *testing.B) { + log.Print("[DEBUG] connect to mongo test instance") + srv, err := mongo.NewServerWithURL(os.Getenv("MONGO_TEST"), 10*time.Second) + assert.Nil(b, err, "failed to dial") + collName := fmt.Sprintf("test_%d", time.Now().Nanosecond()) + conn := mongo.NewConnection(srv, "test", collName) + + data := "" + for i := 0; i < 1000; i++ { + data += "x" + } + lc, err := NewMongoCache(conn) + require.Nil(b, err) + res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { + return []byte(data), nil + }) + require.Nil(b, err) + require.True(b, strings.HasPrefix(string(res), "xxxx"), string(res)) + + key := NewKey("site").ID("key").Scopes("s1", "s2") + b.ResetTimer() + for i := 0; i < b.N; i++ { + lc.Get(key, func() ([]byte, error) { + b.Fail() + return nil, nil + }) + } +} diff --git a/backend/app/rest/cache/options.go b/backend/app/rest/cache/options.go index 89a9c5a3..f0911eb2 100644 --- a/backend/app/rest/cache/options.go +++ b/backend/app/rest/cache/options.go @@ -1,50 +1,35 @@ package cache -import "github.com/pkg/errors" - // Option func type -type Option func(lc *memoryCache) error +type Option func(lc cacheWithOpts) error // MaxValSize functional option defines the largest value's size allowed to be cached // By default it is 0, which means unlimited. func MaxValSize(max int) Option { - return func(lc *memoryCache) error { - lc.maxValueSize = max - if max <= 0 { - return errors.Errorf("negative size for MaxValSize, %d", max) - } - return nil + return func(lc cacheWithOpts) error { + return lc.setMaxValSize(max) } } // MaxKeys functional option defines how many keys to keep. // By default it is 0, which means unlimited. func MaxKeys(max int) Option { - return func(lc *memoryCache) error { - lc.maxKeys = max - if max <= 0 { - return errors.Errorf("negative size for MaxKeys, %d", max) - } - return nil + return func(lc cacheWithOpts) error { + return lc.setMaxKeys(max) } } // MaxCacheSize functional option defines the total size of cached data. // By default it is 0, which means unlimited. func MaxCacheSize(max int64) Option { - return func(lc *memoryCache) error { - lc.maxCacheSize = max - if max <= 0 { - return errors.Errorf("negative size or MaxCacheSize, %d", max) - } - return nil + return func(lc cacheWithOpts) error { + return lc.setMaxCacheSize(max) } } // PostFlushFn functional option defines how callback function called after each Flush. func PostFlushFn(postFlushFn func()) Option { - return func(lc *memoryCache) error { - lc.postFlushFn = postFlushFn - return nil + return func(lc cacheWithOpts) error { + return lc.setPostFlushFn(postFlushFn) } } diff --git a/backend/remark.rest b/backend/remark.rest index 9c8f26dd..6d0e0f97 100644 --- a/backend/remark.rest +++ b/backend/remark.rest @@ -86,9 +86,6 @@ Content-Type: application/json ### list commented posts GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=5 -### get config -GET {{host}}/api/v1/config - ### block user PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=1 diff --git a/backend/vendor/github.com/go-pkgz/repeater/.gitignore b/backend/vendor/github.com/go-pkgz/repeater/.gitignore new file mode 100644 index 00000000..f1c181ec --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/.gitignore @@ -0,0 +1,12 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out diff --git a/backend/vendor/github.com/go-pkgz/repeater/.travis.yml b/backend/vendor/github.com/go-pkgz/repeater/.travis.yml new file mode 100644 index 00000000..baf01e00 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/.travis.yml @@ -0,0 +1,18 @@ +language: go + +go: + - "1.10.x" + +go_import_path: github.com/go-pkgz/repeater + +services: mongodb + +before_install: + - go get github.com/mattn/goveralls + - go get gopkg.in/alecthomas/gometalinter.v2 + - $GOPATH/bin/gometalinter.v2 --install + +script: + - go test ./... + - $GOPATH/bin/gometalinter.v2 --exclude=test --exclude=mock --exclude=vendor ./... + - $GOPATH/bin/goveralls -service=travis-ci diff --git a/backend/vendor/github.com/go-pkgz/repeater/LICENSE b/backend/vendor/github.com/go-pkgz/repeater/LICENSE new file mode 100644 index 00000000..ca125214 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Umputun + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/vendor/github.com/go-pkgz/repeater/README.md b/backend/vendor/github.com/go-pkgz/repeater/README.md new file mode 100644 index 00000000..8b04fc4e --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/README.md @@ -0,0 +1,36 @@ +# Repeater [![Build Status](https://travis-ci.org/go-pkgz/repeater.svg?branch=master)](https://travis-ci.org/go-pkgz/repeater) [![Go Report Card](https://goreportcard.com/badge/github.com/go-pkgz/repeater)](https://goreportcard.com/report/github.com/go-pkgz/repeater) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/repeater/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/repeater?branch=master) + +Repeater calls a function until it returns no error, up to some number of iterations and delays defined by strategy. It terminates immediately on err from the provided (optional) list of critical errors. + +## Install and update + +`go get -u github.com/go-pkgz/repeater` + +## How to use + +New Repeater created by `New(strtg strategy.Interface)` or shortcut for defaults - `NewDefault(repeats int, delay time.Duration) *Repeater`. + +To activate invoke `Do` method. `Do` repeats func until no error returned. Predefined (optional) errors terminates the loop immediately. + +`func (r Repeater) Do(fun func() error, errors ...error) (err error)` + +### Repeating strategy + +User can provide his own strategy implementing the interface: + +```go +type Interface interface { + Start(ctx context.Context) chan struct{} +} +``` + +Returned channels used as "ticks," i.e., for each repeat or initial operation one read from this channel needed. Closing this channel indicates "done with retries." It is pretty much the same idea as `time.Timer` or `time.Tick` implements. Note - the first (technically not-repeated-yet) call won't happen **until something sent to the channel**. For this reason, the typical strategy sends first "tick" before the first wait/sleep. + +Three most common strategies provided by package and ready to use: +1. **Fixed delay**, up to max number of attempts - `NewFixedDelay(repeats int, delay time.Duration)`. +It is the default strategy used by `repeater.NewDefault` constructor +2. **BackOff** with jitter provides exponential backoff. It starts from 100ms interval and goes in steps with `last * math.Pow(factor, attempt)`. Optional jitter randomizes intervals a little bit. The strategy created by `NewBackoff(repeats int, factor float64, jitter bool)`. _Factor = 1 effectively makes this strategy fixed with 100ms delay._ +3. **Once** strategy does not do any repeats and mainly used for tests/mocks - `NewOnce()` + + + diff --git a/backend/vendor/github.com/go-pkgz/repeater/repeater.go b/backend/vendor/github.com/go-pkgz/repeater/repeater.go new file mode 100644 index 00000000..c1721cb3 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/repeater.go @@ -0,0 +1,60 @@ +// Package repeater call fun till it returns no error, up to repeat some number of iterations and delays defined by strategy. +// Repeats number and delays defined by strategy.Interface. Terminates immediately on err from +// provided, optional list of critical errors +package repeater + +import ( + "context" + "time" + + "github.com/go-pkgz/repeater/strategy" +) + +// Repeater is the main object, should be made by New or NewDefault, embeds strategy +type Repeater struct { + strategy.Interface +} + +// New repeater with a given strategy. If strategy=nil initializes with FixedDelay 5sec, 10 times. +func New(strtg strategy.Interface) *Repeater { + if strtg == nil { + strtg = strategy.NewFixedDelay(10, time.Second*5) + } + result := Repeater{Interface: strtg} + return &result +} + +// NewDefault makes repeater with FixedDelay strategy +func NewDefault(repeats int, delay time.Duration) *Repeater { + return New(strategy.NewFixedDelay(repeats, delay)) +} + +// Do repeats fun till no error. Predefined (optional) errors terminate immediately +func (r Repeater) Do(fun func() error, errors ...error) (err error) { + + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() // ensure strategy's channel termination + + inErrors := func(err error) bool { + for _, e := range errors { + if e == err { + return true + } + } + return false + } + + ch := r.Start(ctx) // channel of ticks-like events provided by strategy + + // closed channel indicates completion or early termination, set by strategy + for range ch { + + if err = fun(); err == nil { + return nil + } + if err != nil && inErrors(err) { //terminate on critical error from provided list + return err + } + } + return err +} diff --git a/backend/vendor/github.com/go-pkgz/repeater/strategy/backoff.go b/backend/vendor/github.com/go-pkgz/repeater/strategy/backoff.go new file mode 100644 index 00000000..0f5b8013 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/strategy/backoff.go @@ -0,0 +1,55 @@ +package strategy + +import ( + "context" + "math" + "math/rand" + "time" +) + +// Backoff implements strategy.Interface for exponential-backoff +// it starts from 100ms and goes in steps with last * math.Pow(factor, attempt) +// optional jitter randomize intervals a little bit. +type Backoff struct { + repeats int + factor float64 + jitter bool +} + +// NewBackoff makes Backoff strategy with given factor and optional jitter +func NewBackoff(repeats int, factor float64, jitter bool) Interface { + if repeats == 0 { + repeats = 1 + } + if factor <= 0 { + factor = 1 + } + result := Backoff{repeats: repeats, factor: factor, jitter: jitter} + return &result +} + +// Start returns channel, similar to time.Timer +// then publishing signals to channel ch for retries attempt. Closed ch indicates "done" event +// consumer (repeater) should stop it explicitly after completion +func (b *Backoff) Start(ctx context.Context) (ch chan struct{}) { + ch = make(chan struct{}) + go func() { + defer close(ch) + rnd := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) + minDelay := 100 * time.Millisecond // starts 100ms + for i := 0; i < b.repeats; i++ { + select { + case <-ctx.Done(): + return + default: + ch <- struct{}{} + delay := float64(minDelay) * math.Pow(b.factor, float64(i)) + if b.jitter { + delay = rnd.Float64()*(float64(2*minDelay)) + (delay - float64(minDelay)) + } + time.Sleep(time.Duration(delay)) + } + } + }() + return ch +} diff --git a/backend/vendor/github.com/go-pkgz/repeater/strategy/fixed.go b/backend/vendor/github.com/go-pkgz/repeater/strategy/fixed.go new file mode 100644 index 00000000..adcef013 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/strategy/fixed.go @@ -0,0 +1,41 @@ +package strategy + +import ( + "context" + "time" +) + +// FixedDelay implements strategy.Interface for fixed intervals up to max repeats +type FixedDelay struct { + repeats int + delay time.Duration +} + +// NewFixedDelay makes a Interface +func NewFixedDelay(repeats int, delay time.Duration) Interface { + if repeats == 0 { + repeats = 1 + } + result := FixedDelay{repeats: repeats, delay: delay} + return &result +} + +// Start returns channel, similar to time.Timer +// then publishing signals to channel ch for retries attempt. +// can be terminated (canceled) via context. +func (s *FixedDelay) Start(ctx context.Context) (ch chan struct{}) { + ch = make(chan struct{}) + go func() { + defer close(ch) + for i := 0; i < s.repeats; i++ { + select { + case <-ctx.Done(): + return + default: + ch <- struct{}{} + time.Sleep(s.delay) + } + } + }() + return ch +} diff --git a/backend/vendor/github.com/go-pkgz/repeater/strategy/strategy.go b/backend/vendor/github.com/go-pkgz/repeater/strategy/strategy.go new file mode 100644 index 00000000..5ebb5ebb --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/repeater/strategy/strategy.go @@ -0,0 +1,28 @@ +// Package strategy defines repeater's strategy and implements some. +// Strategy result is a channel acting like time.Timer ot time.Tick +package strategy + +import "context" + +// Interface for repeater strategy. Returns channel with ticks +type Interface interface { + Start(ctx context.Context) chan struct{} +} + +// Once strategy eliminate repeats and makes a single try only +type Once struct{} + +// NewOnce makes no-repeat strategy +func NewOnce() Interface { + return &Once{} +} + +// Start returns closed channel with a single element to prevent any repeats +func (s *Once) Start(ctx context.Context) (ch chan struct{}) { + ch = make(chan struct{}) + go func() { + ch <- struct{}{} + close(ch) + }() + return ch +} From 0f7819ffcf3e4abbffd67bce5ec128044bbe6885 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 24 Jul 2018 21:22:22 -0400 Subject: [PATCH 31/93] add integration test with mongo store, avatar and cache --- backend/app/main_test.go | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/backend/app/main_test.go b/backend/app/main_test.go index d9e5fc0f..436d082f 100644 --- a/backend/app/main_test.go +++ b/backend/app/main_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/globalsign/mgo" + "github.com/go-pkgz/mongo" "github.com/jessevdk/go-flags" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -73,6 +75,60 @@ func TestApplicationDevMode(t *testing.T) { app.Wait() } +func TestApplicationWithMongo(t *testing.T) { + + mongoURL := os.Getenv("MONGO_TEST") + if mongoURL == "" { + mongoURL = "mongodb://localhost:27017/test" + } + if mongoURL == "skip" { + t.Skip("skip mongo app test") + } + + opts := Opts{} + // prepare options + p := flags.NewParser(&opts, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com", + "--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) + require.Nil(t, err) + opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" + opts.BackupLocation = "/tmp" + + // create app + app, err := New(opts) + require.Nil(t, err) + + defer func() { + s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second) + assert.NoError(t, err) + conn := mongo.NewConnection(s, "test_remark", "") + _ = conn.WithDB(func(dbase *mgo.Database) error { + assert.NoError(t, dbase.DropDatabase()) + return nil + }) + }() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Second) + log.Print("[TEST] terminate app") + cancel() + }() + go func() { _ = app.Run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:12345/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + app.Wait() +} + func TestApplicationFailed(t *testing.T) { opts := Opts{} p := flags.NewParser(&opts, flags.Default) From a817a22471c726a4be4355e157df77dd00ac35b1 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 24 Jul 2018 22:14:09 -0400 Subject: [PATCH 32/93] more mongo cache tests --- backend/app/rest/cache/mongo_test.go | 90 ++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/backend/app/rest/cache/mongo_test.go b/backend/app/rest/cache/mongo_test.go index be2a7e4c..aa478020 100644 --- a/backend/app/rest/cache/mongo_test.go +++ b/backend/app/rest/cache/mongo_test.go @@ -124,6 +124,39 @@ func TestMongoCache_MaxKeys(t *testing.T) { }) } +func TestMongoCache_MaxValueSize(t *testing.T) { + conn, err := mongo.MakeTestConnection(t) + assert.NoError(t, err) + defer mongo.RemoveTestCollections(t, conn, "cache") + lc, err := NewMongoCache(conn, MaxKeys(5), MaxValSize(10)) + require.Nil(t, err) + + // put good size value to cache and make sure it cached + res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { + return []byte("result-Z"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res)) + + res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) { + return []byte("result-Zzzz"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-Z", string(res), "got cached value") + + // put too big value to cache and make sure it is not cached + res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { + return []byte("1234567890"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "1234567890", string(res)) + + res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) { + return []byte("result-big"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "result-big", string(res), "got not cached value") +} func TestMongoCache_Parallel(t *testing.T) { var coldCalls int32 conn, err := mongo.MakeTestConnection(t) @@ -171,15 +204,6 @@ func TestMongoCache_Flush(t *testing.T) { require.Equal(t, "value"+id, string(res)) } - cacheSize := func() (count int) { - conn.WithCustomCollection("cache", func(coll *mgo.Collection) (e error) { - count, e = coll.Find(bson.M{"site": "site"}).Count() - require.NoError(t, e) - return e - }) - return count - } - init := func() { lc.Flush(Flusher("site")) addToCache("key1", "s1", "s2") @@ -189,7 +213,7 @@ func TestMongoCache_Flush(t *testing.T) { addToCache("key5", "s2") addToCache("key6") addToCache("key7", "s4", "s3") - require.Equal(t, 7, cacheSize(), "cache init") + require.Equal(t, 7, mongoCacheSize(t, conn), "cache init") } tbl := []struct { @@ -210,10 +234,45 @@ func TestMongoCache_Flush(t *testing.T) { for i, tt := range tbl { init() lc.Flush(Flusher("site").Scopes(tt.scopes...)) - assert.Equal(t, tt.left, cacheSize(), "keys size, %s #%d", tt.msg, i) + assert.Equal(t, tt.left, mongoCacheSize(t, conn), "keys size, %s #%d", tt.msg, i) } } +func TestMongoCache_Scopes(t *testing.T) { + conn, err := mongo.MakeTestConnection(t) + assert.NoError(t, err) + defer mongo.RemoveTestCollections(t, conn, "cache") + lc, err := NewMongoCache(conn) + require.Nil(t, err) + + res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { + return []byte("value"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value", string(res)) + + res, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { + return []byte("value2"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value2", string(res)) + + assert.Equal(t, 2, mongoCacheSize(t, conn)) + lc.Flush(Flusher("site").Scopes("s1")) + assert.Equal(t, 1, mongoCacheSize(t, conn)) + + _, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) { + assert.Fail(t, "should stay") + return nil, nil + }) + assert.Nil(t, err) + res, err = lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) { + return []byte("value-upd"), nil + }) + assert.Nil(t, err) + assert.Equal(t, "value-upd", string(res), "was deleted, update") +} + func BenchmarkMongoCache(b *testing.B) { log.Print("[DEBUG] connect to mongo test instance") srv, err := mongo.NewServerWithURL(os.Getenv("MONGO_TEST"), 10*time.Second) @@ -242,3 +301,12 @@ func BenchmarkMongoCache(b *testing.B) { }) } } + +func mongoCacheSize(t *testing.T, conn *mongo.Connection) (count int) { + conn.WithCustomCollection("cache", func(coll *mgo.Collection) (e error) { + count, e = coll.Find(bson.M{"site": "site"}).Count() + require.NoError(t, e) + return e + }) + return count +} From 5c9d0d611917097f6fe122d174ae7fad5583c17e Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 25 Jul 2018 11:54:17 +0300 Subject: [PATCH 33/93] Fix incorrect edit comment countdown timer calculation #74 --- web/app/common/fetcher.js | 10 +++++++++- web/app/components/comment/comment.jsx | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/web/app/common/fetcher.js b/web/app/common/fetcher.js index 39285155..a19f8430 100644 --- a/web/app/common/fetcher.js +++ b/web/app/common/fetcher.js @@ -2,6 +2,7 @@ import axios from 'axios'; import { BASE_URL, API_BASE } from './constants'; import { siteId } from './settings'; +import store from './store'; const fetcher = {}; const methods = ['get', 'post', 'put', 'patch', 'delete', 'head']; @@ -35,7 +36,14 @@ methods.forEach(method => { } axios(parameters) - .then(res => resolve(res.data)) + .then(res => { + const date = ('date' in res.headers && res.headers.date) || ''; + const timestamp = isNaN(Date.parse(date)) ? 0 : Date.parse(date); + const timeDiff = (new Date() - timestamp) / 1000; + store.set('serverClientTimeDiff', timeDiff); + + resolve(res.data); + }) .catch(error => reject(error)); }); }; diff --git a/web/app/components/comment/comment.jsx b/web/app/components/comment/comment.jsx index 74b139d5..db7b1576 100644 --- a/web/app/components/comment/comment.jsx +++ b/web/app/components/comment/comment.jsx @@ -83,7 +83,8 @@ export default class Comment extends Component { if (userId === commentUserId) { const editDuration = store.get('config') && store.get('config').edit_duration; - const getEditTimeLeft = () => Math.floor(editDuration - (new Date() - new Date(data.time)) / 1000); + const timeDiff = store.get('serverClientTimeDiff') || 0; + const getEditTimeLeft = () => Math.floor(editDuration - ((new Date() - new Date(data.time)) / 1000 - timeDiff)); if (getEditTimeLeft() > 0) { this.editTimerInterval = setInterval(() => { From 1c97e0d31f0f51b37e00398286ac0789bd8c48db Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 25 Jul 2018 12:48:57 -0400 Subject: [PATCH 34/93] add delete avatar and call it from deleteMe controller --- backend/app/rest/api/admin.go | 11 +++++++++++ backend/app/rest/api/rest.go | 1 + backend/app/store/avatar/gridfs.go | 14 +++++++++++++- backend/app/store/avatar/gridfs_test.go | 15 +++++++++++++++ backend/app/store/avatar/localfs.go | 7 +++++++ backend/app/store/avatar/localfs_test.go | 16 ++++++++++++++++ backend/app/store/avatar/store.go | 4 +++- 7 files changed, 66 insertions(+), 2 deletions(-) diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index c1fd52e8..f979aec3 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -7,11 +7,13 @@ import ( "io" "log" "net/http" + "path" "time" "github.com/go-chi/chi" "github.com/go-chi/render" "github.com/umputun/remark/backend/app/rest/auth" + "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" @@ -27,6 +29,7 @@ type admin struct { cache cache.LoadingCache authenticator auth.Authenticator readOnlyAge int + avatarProxy *proxy.Avatar } func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router { @@ -118,6 +121,14 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user") return } + + if claims.User.Picture != "" { + if err := a.avatarProxy.Store.Remove(path.Base(claims.User.Picture)); err != nil { + rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar") + return + } + } + a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, "last")) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID}) diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 8d27e2a5..8b6bd5a1 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -115,6 +115,7 @@ func (s *Rest) routes() chi.Router { cache: s.Cache, authenticator: s.Authenticator, readOnlyAge: s.ReadOnlyAge, + avatarProxy: s.AvatarProxy, } ipFn := func(ip string) string { return store.HashValue(ip, s.DataService.Secret)[:12] } // logger uses it for anonymization diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index 33106f5d..f6103f1c 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -73,7 +73,7 @@ func (gf *GridFS) ID(avatar string) (id string) { return errors.Wrapf(e, "can't open avatar %s", avatar) } id = fh.MD5() - return nil + return errors.Wrapf(fh.Close(), "can't close avatar") }) if err != nil { log.Printf("[DEBUG] can't get file info '%s', %s", avatar, err) @@ -81,3 +81,15 @@ func (gf *GridFS) ID(avatar string) (id string) { } return id } + +// Remove avatar from gridfs +func (gf *GridFS) Remove(avatar string) error { + return gf.Connection.WithDB(func(dbase *mgo.Database) error { + fh, e := dbase.GridFS("fs").Open(avatar) + if e != nil { + return errors.Wrapf(e, "can't get avatar %s", avatar) + } + _ = fh.Close() + return dbase.GridFS("fs").Remove(avatar) + }) +} diff --git a/backend/app/store/avatar/gridfs_test.go b/backend/app/store/avatar/gridfs_test.go index bcc6baad..0ba335f0 100644 --- a/backend/app/store/avatar/gridfs_test.go +++ b/backend/app/store/avatar/gridfs_test.go @@ -34,6 +34,21 @@ func TestGridFS_PutAndGet(t *testing.T) { assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id") } +func TestGridFS_Remove(t *testing.T) { + p, skip := prepGFStore(t) + if skip { + return + } + + assert.NotNil(t, p.Remove("no-such-thing.image")) + + avatar, err := p.Put("user1", strings.NewReader("some picture bin data")) + require.Nil(t, err) + assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) + assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one") + assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed") +} + func prepGFStore(t *testing.T) (Store, bool) { conn, err := mongo.MakeTestConnection(t) if err != nil { diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go index 735fa1bf..1c077b0f 100644 --- a/backend/app/store/avatar/localfs.go +++ b/backend/app/store/avatar/localfs.go @@ -88,6 +88,13 @@ func (fs *LocalFS) ID(avatar string) (id string) { return store.EncodeID(avatar + strconv.FormatInt(fi.ModTime().Unix(), 10)) } +// Remove avatar file +func (fs *LocalFS) Remove(avatar string) error { + location := fs.location(strings.TrimSuffix(avatar, imgSfx)) + avFile := path.Join(location, avatar) + return os.Remove(avFile) +} + // get location (directory) for user id by adding partition to final path in order to keep files // in different subdirectories and avoid too many files in a single place. // the end result is a full path like this - /tmp/avatars.test/92 diff --git a/backend/app/store/avatar/localfs_test.go b/backend/app/store/avatar/localfs_test.go index f60bd9ed..51833f30 100644 --- a/backend/app/store/avatar/localfs_test.go +++ b/backend/app/store/avatar/localfs_test.go @@ -110,6 +110,22 @@ func TestAvatarStoreFS_ID(t *testing.T) { assert.Equal(t, "325d5b451f32c2f8e7f30a9fd65bff6a42954d9a", id) // store.EncodeID("b3daa77b4c04a9551b8781d03191fe098f325e67.image1500000000") } +func TestAvatarStoreFS_Remove(t *testing.T) { + p := NewLocalFS("/tmp/avatars.test", 300) + err := os.MkdirAll("/tmp/avatars.test/30", 0700) + require.NoError(t, err) + defer os.RemoveAll("/tmp/avatars.test") + + assert.NotNil(t, p.Remove("no-such-avatar"), "remove non-existing avatar") + err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666) + require.NoError(t, err) + + assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image")) + _, err = os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image") + assert.NotNil(t, err, "removed for real") + t.Log(err) +} + func BenchmarkAvatarStoreFS_ID(b *testing.B) { p := NewLocalFS("/tmp/avatars.test", 300) os.MkdirAll("/tmp/avatars.test/30", 0700) diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go index 1b612bcf..94392405 100644 --- a/backend/app/store/avatar/store.go +++ b/backend/app/store/avatar/store.go @@ -22,9 +22,11 @@ const imgSfx = ".image" // Store defines interface to store and and load avatars type Store interface { - Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the given reader and return base name + Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader ID(avatarID string) (id string) // unique id of stored avatar's data + Remove(avatarID string) error // remove avatar data + } // resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side From cd7e9832d2fca37401ded46e54a20f25fb78d60f Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 25 Jul 2018 13:17:42 -0400 Subject: [PATCH 35/93] add avatart removal path to deleteMe test --- backend/app/rest/api/admin.go | 4 ++-- backend/app/rest/api/admin_test.go | 9 ++++++++- backend/app/rest/api/middleware_test.go | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index f979aec3..814a1323 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -12,12 +12,12 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" - "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" + "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" + "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/service" ) diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index d3cc64ec..a5805783 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/dgrijalva/jwt-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/store" ) @@ -495,11 +497,16 @@ func TestAdmin_DeleteMeRequest(t *testing.T) { ExpiresAt: time.Now().Add(30 * time.Minute).Unix(), }, User: &store.User{ - ID: "user1", + ID: "user1", + Picture: "pic.image", }, } claims.Flags.DeleteMe = true + _ = os.MkdirAll("/tmp/42", 0700) + defer os.RemoveAll("/tmp/42") + ioutil.WriteFile("/tmp/42/pic.image", []byte("some image data"), 0600) + token, err := srv.Authenticator.JWTService.Token(&claims) assert.Nil(t, err) diff --git a/backend/app/rest/api/middleware_test.go b/backend/app/rest/api/middleware_test.go index 22e6a53d..aa10fb01 100644 --- a/backend/app/rest/api/middleware_test.go +++ b/backend/app/rest/api/middleware_test.go @@ -9,10 +9,10 @@ import ( "github.com/go-chi/chi" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/store" - - "github.com/stretchr/testify/require" ) func TestMiddleware_AppInfo(t *testing.T) { From 6d82a1be93088e7e79a85d3109cdd8ac5229b902 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 25 Jul 2018 13:28:15 -0400 Subject: [PATCH 36/93] extract common hasing with failback --- backend/app/store/user.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/app/store/user.go b/backend/app/store/user.go index a1d032ed..eec073e0 100644 --- a/backend/app/store/user.go +++ b/backend/app/store/user.go @@ -5,6 +5,7 @@ import ( "crypto/sha1" "encoding/hex" "fmt" + "hash" "hash/crc64" "io" "log" @@ -36,12 +37,7 @@ func HashValue(val string, secret string) string { } key := []byte(secret) h := hmac.New(sha1.New, key) - if _, err := io.WriteString(h, val); err != nil { - // fail back to crc64 - log.Printf("[WARN] can't hash ip, %s", err) - return fmt.Sprintf("%x", crc64.Checksum([]byte(val), crc64.MakeTable(crc64.ECMA))) - } - return hex.EncodeToString(h.Sum(nil)) + return hashWithFailback(h, val) } // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases @@ -51,10 +47,15 @@ func EncodeID(id string) string { return id // already hashed or empty } h := sha1.New() - if _, err := io.WriteString(h, id); err != nil { + return hashWithFailback(h, id) +} + +// hashWithFailback tries to has val with hash.Hash and failback to crc if needed +func hashWithFailback(h hash.Hash, val string) string { + if _, err := io.WriteString(h, val); err != nil { // fail back to crc64 - log.Printf("[WARN] can't hash id %s, %s", id, err) - return fmt.Sprintf("%x", crc64.Checksum([]byte(id), crc64.MakeTable(crc64.ECMA))) + log.Printf("[WARN] can't hash id %s, %s", val, err) + return fmt.Sprintf("%x", crc64.Checksum([]byte(val), crc64.MakeTable(crc64.ECMA))) } return hex.EncodeToString(h.Sum(nil)) } From 2a7d23c2cab232c81eceacc1b69656ed25fbb279 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 1 Aug 2018 22:19:06 -0500 Subject: [PATCH 37/93] clean test bolt after remark migrator tests --- backend/app/migrator/remark_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/app/migrator/remark_test.go b/backend/app/migrator/remark_test.go index 384a65c8..414ec90f 100644 --- a/backend/app/migrator/remark_test.go +++ b/backend/app/migrator/remark_test.go @@ -18,6 +18,7 @@ import ( var testDb = "/tmp/test-remark.db" func TestRemark_Export(t *testing.T) { + defer os.Remove(testDb) b := prep(t) r := Remark{DataStore: b} @@ -34,6 +35,8 @@ func TestRemark_Export(t *testing.T) { } func TestRemark_Import(t *testing.T) { + defer os.Remove(testDb) + r1 := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" r2 := `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n" @@ -60,6 +63,8 @@ func TestRemark_Import(t *testing.T) { } func TestRemark_ImportManyWithError(t *testing.T) { + defer os.Remove(testDb) + goodRec := `{"id":"%d","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" buf := &bytes.Buffer{} From e79b95de0c9fe03ca6d45ba504b04203f8fe0117 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 2 Aug 2018 19:26:34 -0500 Subject: [PATCH 38/93] Feature/cors (#183) * add cors internally #160 * revendor --- backend/Gopkg.lock | 135 ++++++- backend/app/rest/api/rest.go | 11 + .../vendor/github.com/go-chi/cors/README.md | 32 ++ backend/vendor/github.com/go-chi/cors/cors.go | 369 ++++++++++++++++++ .../vendor/github.com/go-chi/cors/utils.go | 70 ++++ 5 files changed, 605 insertions(+), 12 deletions(-) create mode 100644 backend/vendor/github.com/go-chi/cors/README.md create mode 100644 backend/vendor/github.com/go-chi/cors/cors.go create mode 100644 backend/vendor/github.com/go-chi/cors/utils.go diff --git a/backend/Gopkg.lock b/backend/Gopkg.lock index 6c9c2cf2..c07a44c5 100644 --- a/backend/Gopkg.lock +++ b/backend/Gopkg.lock @@ -2,218 +2,287 @@ [[projects]] + digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd" name = "cloud.google.com/go" packages = ["compute/metadata"] + pruneopts = "UT" revision = "767c40d6a2e058483c25fa193e963a22da17236d" version = "v0.18.0" [[projects]] + digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9" name = "github.com/PuerkitoBio/goquery" packages = ["."] + pruneopts = "UT" revision = "a86ea073017a6beddef78c8659e7224e8ca634b0" version = "v1.4.0" [[projects]] + digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11" name = "github.com/andybalholm/cascadia" packages = ["."] + pruneopts = "UT" revision = "901648c87902174f774fac311d7f176f8647bdaa" version = "v1.0.0" [[projects]] + digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e" name = "github.com/coreos/bbolt" packages = ["."] + pruneopts = "UT" revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9" version = "v1.3.0" [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55" name = "github.com/dgrijalva/jwt-go" packages = ["."] + pruneopts = "UT" revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e" version = "v3.2.0" [[projects]] + digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5" name = "github.com/didip/tollbooth" packages = [ ".", "errors", "libstring", - "limiter" + "limiter", ] + pruneopts = "UT" revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea" version = "v4.0.0" [[projects]] branch = "master" + digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642" name = "github.com/didip/tollbooth_chi" packages = ["."] + pruneopts = "UT" revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238" [[projects]] - branch = "master" + digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d" name = "github.com/globalsign/mgo" packages = [ ".", "bson", "internal/json", "internal/sasl", - "internal/scram" + "internal/scram", ] + pruneopts = "UT" revision = "113d3961e7311526535a1ef7042196563d442761" + version = "r2018.06.15" [[projects]] + digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07" name = "github.com/go-chi/chi" packages = [ ".", - "middleware" + "middleware", ] + pruneopts = "UT" revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35" version = "v3.3.2" [[projects]] + digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8" + name = "github.com/go-chi/cors" + packages = ["."] + pruneopts = "UT" + revision = "dba6525398619dead495962a916728e7ee2ca322" + version = "v1.0.0" + +[[projects]] + digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab" name = "github.com/go-chi/render" packages = ["."] + pruneopts = "UT" revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519" version = "v1.0.0" [[projects]] + digest = "1:1212e114344a5cdcc834ea69e19d456eef230f9784659080fee67e02ba2cb574" name = "github.com/go-pkgz/mongo" packages = ["."] + pruneopts = "UT" revision = "9a09a574c336c6ae2338a65bbebed2baab2a713c" version = "v1.0.0" [[projects]] + digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7" name = "github.com/go-pkgz/repeater" packages = [ ".", - "strategy" + "strategy", ] + pruneopts = "UT" revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b" version = "v1.0.0" [[projects]] + digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91" name = "github.com/golang/protobuf" packages = ["proto"] + pruneopts = "UT" revision = "925541529c1fa6821df4e44ce2723319eb2be768" version = "v1.0.0" [[projects]] + digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b" name = "github.com/google/uuid" packages = ["."] + pruneopts = "UT" revision = "064e2069ce9c359c118179501254f67d7d37ba24" version = "0.2" [[projects]] + digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce" name = "github.com/gorilla/feeds" packages = ["."] + pruneopts = "UT" revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563" name = "github.com/hashicorp/errwrap" packages = ["."] + pruneopts = "UT" revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" [[projects]] branch = "master" + digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463" name = "github.com/hashicorp/go-multierror" packages = ["."] + pruneopts = "UT" revision = "b7773ae218740a7be65057fc60b366a49b538a44" [[projects]] branch = "master" + digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db" name = "github.com/hashicorp/golang-lru" packages = [ ".", - "simplelru" + "simplelru", ] + pruneopts = "UT" revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" [[projects]] branch = "master" + digest = "1:ce3f7860fd68bd2dd4c3735e2aed8c9de7c7d05bd6ad7d97a6bedcf4fe7b84fb" name = "github.com/hashicorp/logutils" packages = ["."] + pruneopts = "UT" revision = "0dc08b1671f34c4250ce212759ebd880f743d883" [[projects]] branch = "master" + digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab" name = "github.com/jessevdk/go-flags" packages = ["."] + pruneopts = "UT" revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb" [[projects]] branch = "master" + digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed" name = "github.com/microcosm-cc/bluemonday" packages = ["."] + pruneopts = "UT" revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b" [[projects]] branch = "master" + digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291" name = "github.com/nullrocks/identicon" packages = ["."] + pruneopts = "UT" revision = "7875f45b0022edded6377e40639d8aa620193a62" [[projects]] + digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625" name = "github.com/patrickmn/go-cache" packages = ["."] + pruneopts = "UT" revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0" version = "v2.1.0" [[projects]] + digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "UT" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855" name = "github.com/shurcooL/sanitized_anchor_name" packages = ["."] + pruneopts = "UT" revision = "86672fcb3f950f35f2e675df2240550f2a50762f" [[projects]] + digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6" name = "github.com/stretchr/testify" packages = [ "assert", - "require" + "require", ] + pruneopts = "UT" revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" version = "v1.2.1" [[projects]] branch = "master" + digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7" name = "golang.org/x/image" packages = [ "draw", - "math/f64" + "math/f64", ] + pruneopts = "UT" revision = "af66defab954cb421ca110193eed9477c8541e2a" [[projects]] branch = "master" + digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f" name = "golang.org/x/net" packages = [ "context", "context/ctxhttp", "html", - "html/atom" + "html/atom", ] + pruneopts = "UT" revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904" [[projects]] branch = "master" + digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8" name = "golang.org/x/oauth2" packages = [ ".", @@ -223,23 +292,29 @@ "internal", "jws", "jwt", - "yandex" + "yandex", ] + pruneopts = "UT" revision = "543e37812f10c46c622c9575afd7ad22f22a12ba" [[projects]] branch = "master" + digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67" name = "golang.org/x/sys" packages = ["unix"] + pruneopts = "UT" revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" [[projects]] branch = "master" + digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7" name = "golang.org/x/time" packages = ["rate"] + pruneopts = "UT" revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7" [[projects]] + digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547" name = "google.golang.org/appengine" packages = [ ".", @@ -251,20 +326,56 @@ "internal/modules", "internal/remote_api", "internal/urlfetch", - "urlfetch" + "urlfetch", ] + pruneopts = "UT" revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" version = "v1.0.0" [[projects]] + digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db" name = "gopkg.in/russross/blackfriday.v2" packages = ["."] + pruneopts = "UT" revision = "cadec560ec52d93835bf2f15bd794700d3a2473b" version = "v2.0.0" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "da9242a76135f9ee04287bf437ffa42708fc36ec54f39c74127f41a3473f24ac" + input-imports = [ + "github.com/PuerkitoBio/goquery", + "github.com/coreos/bbolt", + "github.com/dgrijalva/jwt-go", + "github.com/didip/tollbooth", + "github.com/didip/tollbooth_chi", + "github.com/globalsign/mgo", + "github.com/globalsign/mgo/bson", + "github.com/go-chi/chi", + "github.com/go-chi/chi/middleware", + "github.com/go-chi/cors", + "github.com/go-chi/render", + "github.com/go-pkgz/mongo", + "github.com/go-pkgz/repeater", + "github.com/google/uuid", + "github.com/gorilla/feeds", + "github.com/hashicorp/go-multierror", + "github.com/hashicorp/golang-lru", + "github.com/hashicorp/logutils", + "github.com/jessevdk/go-flags", + "github.com/microcosm-cc/bluemonday", + "github.com/nullrocks/identicon", + "github.com/patrickmn/go-cache", + "github.com/pkg/errors", + "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", + "golang.org/x/image/draw", + "golang.org/x/oauth2", + "golang.org/x/oauth2/facebook", + "golang.org/x/oauth2/github", + "golang.org/x/oauth2/google", + "golang.org/x/oauth2/yandex", + "gopkg.in/russross/blackfriday.v2", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 8b6bd5a1..ddef456d 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -17,6 +17,7 @@ import ( "github.com/didip/tollbooth_chi" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" + "github.com/go-chi/cors" "github.com/go-chi/render" "github.com/pkg/errors" "gopkg.in/russross/blackfriday.v2" @@ -118,6 +119,16 @@ func (s *Rest) routes() chi.Router { avatarProxy: s.AvatarProxy, } + corsMiddleware := cors.New(cors.Options{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"}, + ExposedHeaders: []string{"Link"}, + AllowCredentials: true, + MaxAge: 300, + }) + router.Use(corsMiddleware.Handler) + ipFn := func(ip string) string { return store.HashValue(ip, s.DataService.Secret)[:12] } // logger uses it for anonymization // auth routes for all providers diff --git a/backend/vendor/github.com/go-chi/cors/README.md b/backend/vendor/github.com/go-chi/cors/README.md new file mode 100644 index 00000000..6e865be3 --- /dev/null +++ b/backend/vendor/github.com/go-chi/cors/README.md @@ -0,0 +1,32 @@ +# CORS net/http middleware + +(fork of github.com/rs/cors) + +## Usage + +```go +func main() { + r := chi.NewRouter() + + // Basic CORS + // for more ideas, see: https://developer.github.com/v3/#cross-origin-resource-sharing + cors := cors.New(cors.Options{ + // AllowedOrigins: []string{"https://foo.com"}, // Use this to allow specific origin hosts + AllowedOrigins: []string{"*"}, + // AllowOriginFunc: func(r *http.Request, origin string) bool { return true }, + AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"}, + ExposedHeaders: []string{"Link"}, + AllowCredentials: true, + MaxAge: 300, // Maximum value not ignored by any of major browsers + }) + r.Use(cors.Handler) + + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("welcome")) + }) + + http.ListenAndServe(":3000", r) +} +``` + diff --git a/backend/vendor/github.com/go-chi/cors/cors.go b/backend/vendor/github.com/go-chi/cors/cors.go new file mode 100644 index 00000000..d85d86cb --- /dev/null +++ b/backend/vendor/github.com/go-chi/cors/cors.go @@ -0,0 +1,369 @@ +// cors package is net/http handler to handle CORS related requests +// as defined by http://www.w3.org/TR/cors/ +// +// You can configure it by passing an option struct to cors.New: +// +// c := cors.New(cors.Options{ +// AllowedOrigins: []string{"foo.com"}, +// AllowedMethods: []string{"GET", "POST", "DELETE"}, +// AllowCredentials: true, +// }) +// +// Then insert the handler in the chain: +// +// handler = c.Handler(handler) +// +// See Options documentation for more options. +// +// The resulting handler is a standard net/http handler. +package cors + +import ( + "log" + "net/http" + "os" + "strconv" + "strings" +) + +// Options is a configuration container to setup the CORS middleware. +type Options struct { + // AllowedOrigins is a list of origins a cross-domain request can be executed from. + // If the special "*" value is present in the list, all origins will be allowed. + // An origin may contain a wildcard (*) to replace 0 or more characters + // (i.e.: http://*.domain.com). Usage of wildcards implies a small performance penality. + // Only one wildcard can be used per origin. + // Default value is ["*"] + AllowedOrigins []string + + // AllowOriginFunc is a custom function to validate the origin. It take the origin + // as argument and returns true if allowed or false otherwise. If this option is + // set, the content of AllowedOrigins is ignored. + AllowOriginFunc func(r *http.Request, origin string) bool + + // AllowedMethods is a list of methods the client is allowed to use with + // cross-domain requests. Default value is simple methods (GET and POST) + AllowedMethods []string + + // AllowedHeaders is list of non simple headers the client is allowed to use with + // cross-domain requests. + // If the special "*" value is present in the list, all headers will be allowed. + // Default value is [] but "Origin" is always appended to the list. + AllowedHeaders []string + + // ExposedHeaders indicates which headers are safe to expose to the API of a CORS + // API specification + ExposedHeaders []string + + // AllowCredentials indicates whether the request can include user credentials like + // cookies, HTTP authentication or client side SSL certificates. + AllowCredentials bool + + // MaxAge indicates how long (in seconds) the results of a preflight request + // can be cached + MaxAge int + + // OptionsPassthrough instructs preflight to let other potential next handlers to + // process the OPTIONS method. Turn this on if your application handles OPTIONS. + OptionsPassthrough bool + + // Debugging flag adds additional output to debug server side CORS issues + Debug bool +} + +// Cors http handler +type Cors struct { + // Debug logger + log *log.Logger + + // Set to true when allowed origins contains a "*" + allowedOriginsAll bool + + // Normalized list of plain allowed origins + allowedOrigins []string + + // List of allowed origins containing wildcards + allowedWOrigins []wildcard + + // Optional origin validator function + allowOriginFunc func(r *http.Request, origin string) bool + + // Set to true when allowed headers contains a "*" + allowedHeadersAll bool + + // Normalized list of allowed headers + allowedHeaders []string + + // Normalized list of allowed methods + allowedMethods []string + + // Normalized list of exposed headers + exposedHeaders []string + allowCredentials bool + maxAge int + optionPassthrough bool +} + +// New creates a new Cors handler with the provided options. +func New(options Options) *Cors { + c := &Cors{ + exposedHeaders: convert(options.ExposedHeaders, http.CanonicalHeaderKey), + allowOriginFunc: options.AllowOriginFunc, + allowCredentials: options.AllowCredentials, + maxAge: options.MaxAge, + optionPassthrough: options.OptionsPassthrough, + } + if options.Debug { + c.log = log.New(os.Stdout, "[cors] ", log.LstdFlags) + } + + // Normalize options + // Note: for origins and methods matching, the spec requires a case-sensitive matching. + // As it may error prone, we chose to ignore the spec here. + + // Allowed Origins + if len(options.AllowedOrigins) == 0 { + // Default is all origins + c.allowedOriginsAll = true + } else { + c.allowedOrigins = []string{} + c.allowedWOrigins = []wildcard{} + for _, origin := range options.AllowedOrigins { + // Normalize + origin = strings.ToLower(origin) + if origin == "*" { + // If "*" is present in the list, turn the whole list into a match all + c.allowedOriginsAll = true + c.allowedOrigins = nil + c.allowedWOrigins = nil + break + } else if i := strings.IndexByte(origin, '*'); i >= 0 { + // Split the origin in two: start and end string without the * + w := wildcard{origin[0:i], origin[i+1 : len(origin)]} + c.allowedWOrigins = append(c.allowedWOrigins, w) + } else { + c.allowedOrigins = append(c.allowedOrigins, origin) + } + } + } + + // Allowed Headers + if len(options.AllowedHeaders) == 0 { + // Use sensible defaults + c.allowedHeaders = []string{"Origin", "Accept", "Content-Type"} + } else { + // Origin is always appended as some browsers will always request for this header at preflight + c.allowedHeaders = convert(append(options.AllowedHeaders, "Origin"), http.CanonicalHeaderKey) + for _, h := range options.AllowedHeaders { + if h == "*" { + c.allowedHeadersAll = true + c.allowedHeaders = nil + break + } + } + } + + // Allowed Methods + if len(options.AllowedMethods) == 0 { + // Default is spec's "simple" methods + c.allowedMethods = []string{"GET", "POST"} + } else { + c.allowedMethods = convert(options.AllowedMethods, strings.ToUpper) + } + + return c +} + +// Default creates a new Cors handler with default options +func Default() *Cors { + return New(Options{}) +} + +// Handler apply the CORS specification on the request, and add relevant CORS headers +// as necessary. +func (c *Cors) Handler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "OPTIONS" { + c.logf("Handler: Preflight request") + c.handlePreflight(w, r) + // Preflight requests are standalone and should stop the chain as some other + // middleware may not handle OPTIONS requests correctly. One typical example + // is authentication middleware ; OPTIONS requests won't carry authentication + // headers (see #1) + if c.optionPassthrough { + next.ServeHTTP(w, r) + } + } else { + c.logf("Handler: Actual request") + c.handleActualRequest(w, r) + next.ServeHTTP(w, r) + } + }) +} + +// handlePreflight handles pre-flight CORS requests +func (c *Cors) handlePreflight(w http.ResponseWriter, r *http.Request) { + headers := w.Header() + origin := r.Header.Get("Origin") + + if r.Method != "OPTIONS" { + c.logf("Preflight aborted: %s!=OPTIONS", r.Method) + return + } + // Always set Vary headers + // see https://github.com/rs/cors/issues/10, + // https://github.com/rs/cors/commit/dbdca4d95feaa7511a46e6f1efb3b3aa505bc43f#commitcomment-12352001 + headers.Add("Vary", "Origin") + headers.Add("Vary", "Access-Control-Request-Method") + headers.Add("Vary", "Access-Control-Request-Headers") + + if origin == "" { + c.logf("Preflight aborted: empty origin") + return + } + if !c.isOriginAllowed(r, origin) { + c.logf("Preflight aborted: origin '%s' not allowed", origin) + return + } + + reqMethod := r.Header.Get("Access-Control-Request-Method") + if !c.isMethodAllowed(reqMethod) { + c.logf("Preflight aborted: method '%s' not allowed", reqMethod) + return + } + reqHeaders := parseHeaderList(r.Header.Get("Access-Control-Request-Headers")) + if !c.areHeadersAllowed(reqHeaders) { + c.logf("Preflight aborted: headers '%v' not allowed", reqHeaders) + return + } + headers.Set("Access-Control-Allow-Origin", origin) + // Spec says: Since the list of methods can be unbounded, simply returning the method indicated + // by Access-Control-Request-Method (if supported) can be enough + headers.Set("Access-Control-Allow-Methods", strings.ToUpper(reqMethod)) + if len(reqHeaders) > 0 { + + // Spec says: Since the list of headers can be unbounded, simply returning supported headers + // from Access-Control-Request-Headers can be enough + headers.Set("Access-Control-Allow-Headers", strings.Join(reqHeaders, ", ")) + } + if c.allowCredentials { + headers.Set("Access-Control-Allow-Credentials", "true") + } + if c.maxAge > 0 { + headers.Set("Access-Control-Max-Age", strconv.Itoa(c.maxAge)) + } + c.logf("Preflight response headers: %v", headers) +} + +// handleActualRequest handles simple cross-origin requests, actual request or redirects +func (c *Cors) handleActualRequest(w http.ResponseWriter, r *http.Request) { + headers := w.Header() + origin := r.Header.Get("Origin") + + if r.Method == "OPTIONS" { + c.logf("Actual request no headers added: method == %s", r.Method) + return + } + // Always set Vary, see https://github.com/rs/cors/issues/10 + headers.Add("Vary", "Origin") + if origin == "" { + c.logf("Actual request no headers added: missing origin") + return + } + if !c.isOriginAllowed(r, origin) { + c.logf("Actual request no headers added: origin '%s' not allowed", origin) + return + } + + // Note that spec does define a way to specifically disallow a simple method like GET or + // POST. Access-Control-Allow-Methods is only used for pre-flight requests and the + // spec doesn't instruct to check the allowed methods for simple cross-origin requests. + // We think it's a nice feature to be able to have control on those methods though. + if !c.isMethodAllowed(r.Method) { + if c.log != nil { + c.logf("Actual request no headers added: method '%s' not allowed", + r.Method) + } + + return + } + headers.Set("Access-Control-Allow-Origin", origin) + if len(c.exposedHeaders) > 0 { + headers.Set("Access-Control-Expose-Headers", strings.Join(c.exposedHeaders, ", ")) + } + if c.allowCredentials { + headers.Set("Access-Control-Allow-Credentials", "true") + } + c.logf("Actual response added headers: %v", headers) +} + +// convenience method. checks if debugging is turned on before printing +func (c *Cors) logf(format string, a ...interface{}) { + if c.log != nil { + c.log.Printf(format, a...) + } +} + +// isOriginAllowed checks if a given origin is allowed to perform cross-domain requests +// on the endpoint +func (c *Cors) isOriginAllowed(r *http.Request, origin string) bool { + if c.allowOriginFunc != nil { + return c.allowOriginFunc(r, origin) + } + if c.allowedOriginsAll { + return true + } + origin = strings.ToLower(origin) + for _, o := range c.allowedOrigins { + if o == origin { + return true + } + } + for _, w := range c.allowedWOrigins { + if w.match(origin) { + return true + } + } + return false +} + +// isMethodAllowed checks if a given method can be used as part of a cross-domain request +// on the endpoing +func (c *Cors) isMethodAllowed(method string) bool { + if len(c.allowedMethods) == 0 { + // If no method allowed, always return false, even for preflight request + return false + } + method = strings.ToUpper(method) + if method == "OPTIONS" { + // Always allow preflight requests + return true + } + for _, m := range c.allowedMethods { + if m == method { + return true + } + } + return false +} + +// areHeadersAllowed checks if a given list of headers are allowed to used within +// a cross-domain request. +func (c *Cors) areHeadersAllowed(requestedHeaders []string) bool { + if c.allowedHeadersAll || len(requestedHeaders) == 0 { + return true + } + for _, header := range requestedHeaders { + header = http.CanonicalHeaderKey(header) + found := false + for _, h := range c.allowedHeaders { + if h == header { + found = true + } + } + if !found { + return false + } + } + return true +} diff --git a/backend/vendor/github.com/go-chi/cors/utils.go b/backend/vendor/github.com/go-chi/cors/utils.go new file mode 100644 index 00000000..cd24831f --- /dev/null +++ b/backend/vendor/github.com/go-chi/cors/utils.go @@ -0,0 +1,70 @@ +package cors + +import "strings" + +const toLower = 'a' - 'A' + +type converter func(string) string + +type wildcard struct { + prefix string + suffix string +} + +func (w wildcard) match(s string) bool { + return len(s) >= len(w.prefix+w.suffix) && strings.HasPrefix(s, w.prefix) && strings.HasSuffix(s, w.suffix) +} + +// convert converts a list of string using the passed converter function +func convert(s []string, c converter) []string { + out := []string{} + for _, i := range s { + out = append(out, c(i)) + } + return out +} + +// parseHeaderList tokenize + normalize a string containing a list of headers +func parseHeaderList(headerList string) []string { + l := len(headerList) + h := make([]byte, 0, l) + upper := true + // Estimate the number headers in order to allocate the right splice size + t := 0 + for i := 0; i < l; i++ { + if headerList[i] == ',' { + t++ + } + } + headers := make([]string, 0, t) + for i := 0; i < l; i++ { + b := headerList[i] + if b >= 'a' && b <= 'z' { + if upper { + h = append(h, b-toLower) + } else { + h = append(h, b) + } + } else if b >= 'A' && b <= 'Z' { + if !upper { + h = append(h, b+toLower) + } else { + h = append(h, b) + } + } else if b == '-' || (b >= '0' && b <= '9') { + h = append(h, b) + } + + if b == ' ' || b == ',' || i == l-1 { + if len(h) > 0 { + // Flush the found header + headers = append(headers, string(h)) + h = h[:0] + upper = true + } + } else { + upper = b == '-' + } + } + return headers +} From 39e426bd4bb27522b13a82270e5d7f499df09912 Mon Sep 17 00:00:00 2001 From: Anton Kosourov Date: Fri, 3 Aug 2018 08:05:53 +0500 Subject: [PATCH 39/93] Add wordpress importer (#182) * add wordpress, test * add wp migrator to rest * add readme, fix test, fix error handling --- README.md | 8 +- backend/app/main.go | 13 +- backend/app/migrator/disqus.go | 12 +- backend/app/migrator/disqus_test.go | 6 +- backend/app/migrator/migrator.go | 2 + backend/app/migrator/migrator_test.go | 30 +++- backend/app/migrator/wordpress.go | 154 ++++++++++++++++ backend/app/migrator/wordpress_test.go | 235 +++++++++++++++++++++++++ backend/app/rest/api/migrator.go | 26 ++- backend/app/rest/api/migrator_test.go | 186 ++++++++++++++++++- backend/scripts/import-wordpress.sh | 5 + 11 files changed, 645 insertions(+), 32 deletions(-) create mode 100644 backend/app/migrator/wordpress.go create mode 100644 backend/app/migrator/wordpress_test.go create mode 100755 backend/scripts/import-wordpress.sh diff --git a/README.md b/README.md index d49d7053..0558e2aa 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * Social login via Google, Facebook, Github and Yandex * Multi-level nested comments with both tree and plain presentations -* Import from disqus +* Import from disqus, wordpress * Markdown support * Moderator can remove comments and block users * Voting, pinning and verification system @@ -162,6 +162,12 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co 2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip .xml.gz`. 3. Run import command - `docker-compose exec remark42 /srv/import-disqus.sh .xml ` +#### Initial import from WordPress + +1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. +2. Move this file to your remark42 host within `./var` +3. Run import command - `docker-compose exec remark42 /srv/import-wordpress.sh .xml ` + #### Backup and restore ##### Automatic backups diff --git a/backend/app/main.go b/backend/app/main.go index 0e1d1aa3..a88b5759 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -196,12 +196,13 @@ func New(opts Opts) (*Application, error) { exporter := &migrator.Remark{DataStore: dataService} migr := &api.Migrator{ - Version: revision, - Cache: loadingCache, - NativeImporter: &migrator.Remark{DataStore: dataService}, - DisqusImporter: &migrator.Disqus{DataStore: dataService}, - NativeExported: &migrator.Remark{DataStore: dataService}, - SecretKey: opts.SecretKey, + Version: revision, + Cache: loadingCache, + NativeImporter: &migrator.Remark{DataStore: dataService}, + DisqusImporter: &migrator.Disqus{DataStore: dataService}, + WordPressImporter: &migrator.WordPress{DataStore: dataService}, + NativeExported: &migrator.Remark{DataStore: dataService}, + SecretKey: opts.SecretKey, } authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go index 547fccda..e9d657b3 100644 --- a/backend/app/migrator/disqus.go +++ b/backend/app/migrator/disqus.go @@ -68,14 +68,14 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) { } if failed > 0 { - return passed, errors.Errorf("failed to save %d comments", failed) + err = errors.Errorf("failed to save %d comments", failed) + if passed == 0 { + err = errors.New("import failed") + } } log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID) - if failed > 0 && passed == 0 { - err = errors.New("import failed") - } return passed, err } @@ -133,7 +133,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { Name: comment.AuthorName, IP: comment.IP, }, - Text: d.cleanText(comment.Message), + Text: cleanText(comment.Message), Timestamp: comment.CreatedAt, ParentID: comment.Pid.Val, } @@ -158,7 +158,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { return commentsCh } -func (d *Disqus) cleanText(text string) string { +func cleanText(text string) string { text = strings.Replace(text, "\n", "", -1) text = strings.Replace(text, "\t", "", -1) return text diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index 96142a5d..af7a55ba 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -21,7 +21,7 @@ func TestDisqus_Import(t *testing.T) { require.Nil(t, err, "create store") dataStore := service.DataStore{Interface: b} d := Disqus{DataStore: &dataStore} - size, err := d.Import(strings.NewReader(xmlTest), "test") + size, err := d.Import(strings.NewReader(xmlTestDisqus), "test") assert.Nil(t, err) assert.Equal(t, 3, size) @@ -49,7 +49,7 @@ func TestDisqus_Import(t *testing.T) { func TestDisqus_Convert(t *testing.T) { d := Disqus{} - ch := d.convert(strings.NewReader(xmlTest), "test") + ch := d.convert(strings.NewReader(xmlTestDisqus), "test") res := []store.Comment{} for comment := range ch { @@ -74,7 +74,7 @@ func TestDisqus_Convert(t *testing.T) { assert.Equal(t, exp0, res[0]) } -var xmlTest = ` +var xmlTestDisqus = ` diff --git a/backend/app/migrator/migrator.go b/backend/app/migrator/migrator.go index 40c156ec..422a25ab 100644 --- a/backend/app/migrator/migrator.go +++ b/backend/app/migrator/migrator.go @@ -46,6 +46,8 @@ func ImportComments(p ImportParams) (int, error) { switch p.Provider { case "disqus": importer = &Disqus{DataStore: p.DataStore} + case "wordpress": + importer = &WordPress{DataStore: p.DataStore} case "native": importer = &Remark{DataStore: p.DataStore} default: diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 8825c53e..120b1185 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -19,7 +19,7 @@ func TestMigrator_ImportDisqus(t *testing.T) { os.Remove("/tmp/disqus-test.xml") }() - err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTest), 0600) + err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600) require.Nil(t, err) b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) @@ -39,6 +39,32 @@ func TestMigrator_ImportDisqus(t *testing.T) { assert.Equal(t, 3, len(last), "3 comments imported") } +func TestMigrator_ImportWordPress(t *testing.T) { + defer func() { + os.Remove("/tmp/remark-test.db") + os.Remove("/tmp/wordpress-test.xml") + }() + + err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600) + require.Nil(t, err) + + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + require.Nil(t, err, "create store") + dataStore := &service.DataStore{Interface: b} + size, err := ImportComments(ImportParams{ + DataStore: dataStore, + InputFile: "/tmp/wordpress-test.xml", + SiteID: "test", + Provider: "wordpress", + }) + assert.Nil(t, err) + assert.Equal(t, 3, size) + + last, err := dataStore.Last("test", 10) + assert.Nil(t, err) + assert.Equal(t, 3, len(last), "3 comments imported") +} + func TestMigrator_ImportRemark(t *testing.T) { defer func() { os.Remove("/tmp/remark-test.db") @@ -70,7 +96,7 @@ func TestMigrator_ImportRemark(t *testing.T) { } func TestMigrator_ImportFailed(t *testing.T) { - + defer os.Remove("/tmp/remark-test.db") b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") dataStore := &service.DataStore{Interface: b} diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go new file mode 100644 index 00000000..362ef881 --- /dev/null +++ b/backend/app/migrator/wordpress.go @@ -0,0 +1,154 @@ +package migrator + +import ( + "encoding/xml" + "html" + "io" + "log" + "time" + + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store" +) + +const wpTimeLayout = "2006-01-02 15:04:05" + +// WordPress implements Importer from WP xml +type WordPress struct { + DataStore Store +} + +type wpItem struct { + Link string `xml:"link"` + Comments []wpComment `xml:"comment"` +} + +type wpComment struct { + ID string `xml:"comment_id"` + Author string `xml:"comment_author"` + AuthorEmail string `xml:"comment_author_email"` + AuthorIP string `xml:"comment_author_IP"` + Date wpTime `xml:"comment_date_gmt"` + Content string `xml:"comment_content"` + Approved string `xml:"comment_approved"` + PID string `xml:"comment_parent"` +} + +type wpTime struct { + time time.Time +} + +func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var v string + if err := d.DecodeElement(&v, &start); err != nil { + return err + } + t, err := time.Parse(wpTimeLayout, v) + if err != nil { + return err + } + w.time = t + return err +} + +// Import comments from WP and save to store +func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) { + + if err = w.DataStore.DeleteAll(siteID); err != nil { + return 0, err + } + + commentsCh := w.convert(r, siteID) + failed, passed := 0, 0 + for c := range commentsCh { + if _, err = w.DataStore.Create(c); err != nil { + failed++ + continue + } + passed++ + } + + if failed > 0 { + err = errors.Errorf("failed to save %d comments", failed) + if passed == 0 { + err = errors.New("import failed") + } + } + + log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID) + + return passed, err +} + +func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { + + decoder := xml.NewDecoder(r) + commentsCh := make(chan store.Comment) + + stats := struct { + inpItems, failedItems int + inpComments, failedComments int + rejectedComments int // not approved + }{} + + go func() { + for { + t, err := decoder.Token() + if t == nil || err != nil { + break + } + + switch el := t.(type) { + case xml.StartElement: + if el.Name.Local == "item" { + stats.inpItems++ + item := wpItem{} + if err := decoder.DecodeElement(&item, &el); err != nil { + log.Printf("[WARN] Can't decode item, %s", err) + stats.failedItems++ + continue + } + if item.Comments != nil { + for _, comment := range item.Comments { + if comment.Approved != "1" { + stats.rejectedComments++ + continue + } + + if comment.PID == "0" { + comment.PID = "" + } + c := store.Comment{ + ID: comment.ID, + Locator: store.Locator{URL: item.Link, SiteID: siteID}, + User: store.User{ + ID: "wordpress_" + store.EncodeID(comment.Author), + Name: comment.Author, + IP: comment.AuthorIP, + }, + Text: w.cleanUnescapeText(comment.Content), // sanitize remains on comment create + Timestamp: comment.Date.time, + ParentID: comment.PID, + } + commentsCh <- c + stats.inpComments++ + if stats.inpComments%1000 == 0 { + log.Printf("[DEBUG] proccessed %d comments", stats.inpComments) + } + } + } + } + } + } + close(commentsCh) + log.Printf("[INFO] converted %d comments, %+v", stats.inpComments-stats.failedComments, stats) + }() + return commentsCh +} + +func (w *WordPress) cleanUnescapeText(text string) string { + text = cleanText(text) + text = html.UnescapeString(text) + return text +} diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go new file mode 100644 index 00000000..c915956a --- /dev/null +++ b/backend/app/migrator/wordpress_test.go @@ -0,0 +1,235 @@ +package migrator + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/coreos/bbolt" + "github.com/stretchr/testify/assert" + + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/service" +) + +func TestWordPress_Import(t *testing.T) { + siteID := "testWP" + defer os.Remove("/tmp/remark-test.db") + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) + assert.Nil(t, err, "create store") + + dataStore := service.DataStore{Interface: b} + wp := WordPress{DataStore: &dataStore} + size, err := wp.Import(strings.NewReader(xmlTestWP), siteID) + assert.Nil(t, err) + assert.Equal(t, 3, size) + + last, err := dataStore.Last(siteID, 10) + assert.Nil(t, err) + assert.Equal(t, 3, len(last), "3 comments imported") + + c := last[0] + assert.Equal(t, "14", c.ID) + assert.Equal(t, store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID}, c.Locator) + assert.Equal(t, "wordpress_75b2b81081f82495d7af26759e67af6554ffda4a", c.User.ID) + assert.Equal(t, "SuperUser3", c.User.Name) + assert.Equal(t, "b646e160768fbc1414d3b2c8f88a767bfbb00871", c.User.IP) + ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14") + assert.Equal(t, ts, c.Timestamp) + assert.Equal(t, c.Text, "Mekkatorque was over in that tent up to the right") + + posts, err := dataStore.List(siteID, 0, 0) + assert.Nil(t, err) + assert.Equal(t, 1, len(posts)) + + p := posts[0] + assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL) + + count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID}) + assert.Nil(t, err) + assert.Equal(t, 3, count) +} + +func TestWordPress_Convert(t *testing.T) { + wp := WordPress{} + ch := wp.convert(strings.NewReader(xmlTestWP), "testWP") + + comments := []store.Comment{} + for c := range ch { + comments = append(comments, c) + } + assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded") + + exp1 := store.Comment{ + ID: "13", + Locator: store.Locator{ + SiteID: "testWP", + URL: "https://realmenweardress.es/2010/07/do-you-rp/", + }, + Text: `[...] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. [...]`, + User: store.User{ + Name: "Wednesday Reading « Cynwise's Battlefield Manual", + ID: "wordpress_" + store.EncodeID("Wednesday Reading « Cynwise's Battlefield Manual"), + IP: "74.200.244.101", + }, + } + exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08") + assert.Equal(t, exp1, comments[1]) +} + +var xmlTestWP = ` + + + + + Real Men Wear Dress.es + https://realmenweardress.es + SuperAdmin's gaming and technological musings + Mon, 23 Jul 2018 10:21:47 +0000 + en-US + 1.2 + https://realmenweardress.es + https://realmenweardress.es + + 2 + 1 + + + 25 + + + + + + + 39 + + + + + https://wordpress.org/?v=4.8.1 + + + Post without comments + https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/ + Sat, 19 Jun 2010 08:34:13 +0000 + + http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg + + + + 6 + + + + + + + 1 + 0 + + + 0 + + + + + + + + Post with comments. One is not approved + https://realmenweardress.es/2010/07/do-you-rp/ + Mon, 19 Jul 2010 14:24:22 +0000 + + http://realmenweardress.es/?p=100 + + I need to stand on things else I can't reachMeet Grokknomel?]]> + + 100 + + + + + + + 0 + 0 + + + 0 + + + + + + + + + + 8 + + + http://superuser1.blogspot.com + + + + + + + 0 + 0 + + + 9 + + + http://thewowstorm.wordpress.com + + + + + + + 0 + 0 + + + 13 + + + http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/ + + + + + + + 0 + 0 + + + 14 + + + http://realmenweardress.es + + + + + + + 13 + 2 + + + + +` diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index c8bba0da..bb891bd2 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -23,12 +23,13 @@ import ( // Migrator rest runs on unexposed port and available for local requests only type Migrator struct { - Version string - Cache cache.LoadingCache - NativeImporter migrator.Importer - DisqusImporter migrator.Importer - NativeExported migrator.Exporter - SecretKey string + Version string + Cache cache.LoadingCache + NativeImporter migrator.Importer + DisqusImporter migrator.Importer + WordPressImporter migrator.Importer + NativeExported migrator.Exporter + SecretKey string httpServer *http.Server lock sync.Mutex @@ -76,7 +77,7 @@ func (m *Migrator) routes() chi.Router { return router } -// POST /import?secret=key&site=site-id&provider=disqus|remark +// POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress // imports comments from post body. func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { @@ -88,10 +89,17 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { } siteID := r.URL.Query().Get("site") - importer := m.NativeImporter - if r.URL.Query().Get("provider") == "disqus" { + + var importer migrator.Importer + switch r.URL.Query().Get("provider") { + case "disqus": importer = m.DisqusImporter + case "wordpress": + importer = m.WordPressImporter + default: + importer = m.NativeImporter } + log.Printf("[DEBUG] import request for site=%s", siteID) size, err := importer.Import(r.Body, siteID) if err != nil { diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 98a34908..43f6c0e4 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -40,6 +40,26 @@ func TestMigrator_Import(t *testing.T) { assert.Equal(t, `{"size":2,"status":"ok"}`+"\n", string(b)) } +func TestMigrator_ImportFromWP(t *testing.T) { + srv, ts := prepImportSrv(t) + assert.NotNil(t, srv) + defer cleanupImportSrv(srv, ts) + + r := strings.NewReader(xmlTestWP) + + client := &http.Client{Timeout: 1 * time.Second} + req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress&secret=123456", r) + assert.Nil(t, err) + req.Header.Add("Content-Type", "application/xml; charset=utf-8") + resp, err := client.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + b, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, `{"size":3,"status":"ok"}`+"\n", string(b)) +} + func TestMigrator_ImportRejected(t *testing.T) { srv, ts := prepImportSrv(t) assert.NotNil(t, srv) @@ -109,11 +129,12 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) { require.Nil(t, err) dataStore := &service.DataStore{Interface: b} svc = &Migrator{ - DisqusImporter: &migrator.Disqus{DataStore: dataStore}, - NativeImporter: &migrator.Remark{DataStore: dataStore}, - NativeExported: &migrator.Remark{DataStore: dataStore}, - Cache: &cache.Nop{}, - SecretKey: "123456", + DisqusImporter: &migrator.Disqus{DataStore: dataStore}, + WordPressImporter: &migrator.WordPress{DataStore: dataStore}, + NativeImporter: &migrator.Remark{DataStore: dataStore}, + NativeExported: &migrator.Remark{DataStore: dataStore}, + Cache: &cache.Nop{}, + SecretKey: "123456", } routes := svc.routes() @@ -125,3 +146,158 @@ func cleanupImportSrv(srv *Migrator, ts *httptest.Server) { ts.Close() os.Remove(testDb) } + +var xmlTestWP = ` + + + + + Real Men Wear Dress.es + https://realmenweardress.es + SuperAdmin's gaming and technological musings + Mon, 23 Jul 2018 10:21:47 +0000 + en-US + 1.2 + https://realmenweardress.es + https://realmenweardress.es + + 2 + 1 + + + 25 + + + + + + + 39 + + + + + https://wordpress.org/?v=4.8.1 + + + Post without comments + https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/ + Sat, 19 Jun 2010 08:34:13 +0000 + + http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg + + + + 6 + + + + + + + 1 + 0 + + + 0 + + + + + + + + Post with comments. One is not approved + https://realmenweardress.es/2010/07/do-you-rp/ + Mon, 19 Jul 2010 14:24:22 +0000 + + http://realmenweardress.es/?p=100 + + I need to stand on things else I can't reachMeet Grokknomel?]]> + + 100 + + + + + + + 0 + 0 + + + 0 + + + + + + + + + + 8 + + + http://superuser1.blogspot.com + + + + + + + 0 + 0 + + + 9 + + + http://thewowstorm.wordpress.com + + + + + + + 0 + 0 + + + 13 + + + http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/ + + + + + + + 0 + 0 + + + 14 + + + http://realmenweardress.es + + + + + + + 13 + 2 + + + + +` diff --git a/backend/scripts/import-wordpress.sh b/backend/scripts/import-wordpress.sh new file mode 100755 index 00000000..1c224379 --- /dev/null +++ b/backend/scripts/import-wordpress.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e +echo "import wordpress file $1 to site $2" +curl -X POST -H "Content-Type: application/xml" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=wordpress&secret=${SECRET}" +echo "import completed" \ No newline at end of file From af3208ac094d82b73807b02c0d74bc2266c2c7ab Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 6 Aug 2018 10:49:44 -0500 Subject: [PATCH 40/93] add info about SITE env --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0558e2aa..5deac438 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ services: container_name: "remark42" environment: - REMARK_URL=https://demo.remark42.com # url pointing to your remark42 server + - SITE=my-site # site ID, same as used for `site_id`, see [Setup on your website](https://github.com/umputun/remark#setup-on-your-website) - SECRET=abcd-123456-xyz-$%^& # secret key - AUTH_GITHUB_CID=12345667890 # oauth2 client ID - AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret From c22ed2300b94ad7762aa13042ef47cb8a7f4516d Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 6 Aug 2018 10:50:31 -0500 Subject: [PATCH 41/93] clear link in md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5deac438..82e56862 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ services: container_name: "remark42" environment: - REMARK_URL=https://demo.remark42.com # url pointing to your remark42 server - - SITE=my-site # site ID, same as used for `site_id`, see [Setup on your website](https://github.com/umputun/remark#setup-on-your-website) + - SITE=my-site # site ID, same as used for `site_id`, see "Setup on your website" - SECRET=abcd-123456-xyz-$%^& # secret key - AUTH_GITHUB_CID=12345667890 # oauth2 client ID - AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret From b17e545773ed93b4906382e4bfe257dff08070cc Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 6 Aug 2018 10:52:35 -0500 Subject: [PATCH 42/93] same placeholder in compose as in widget --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 82e56862..2eb7abf0 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ services: container_name: "remark42" environment: - REMARK_URL=https://demo.remark42.com # url pointing to your remark42 server - - SITE=my-site # site ID, same as used for `site_id`, see "Setup on your website" + - SITE=YOUR_SITE_ID # site ID, same as used for `site_id`, see "Setup on your website" - SECRET=abcd-123456-xyz-$%^& # secret key - AUTH_GITHUB_CID=12345667890 # oauth2 client ID - AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret From 1b9ee3d7633ff06b26da9856d7792fd08246e164 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 8 Aug 2018 14:33:18 -0500 Subject: [PATCH 43/93] feature/keystore (#185) * add keyStore and switch all secret usage to this store - implement static ks - add shared secret * lint: shadow in jwt and missing comment * rename static key group to shared and use top-level shared secret * move siteID extraction (from token) to internal func * lint: merge var and assign for key test --- backend/app/main.go | 50 +++++++++++++++------- backend/app/migrator/disqus_test.go | 5 ++- backend/app/migrator/migrator_test.go | 7 +-- backend/app/migrator/remark_test.go | 7 +-- backend/app/migrator/wordpress_test.go | 5 ++- backend/app/rest/api/migrator.go | 28 +++++++++--- backend/app/rest/api/migrator_test.go | 7 +-- backend/app/rest/api/rest.go | 3 +- backend/app/rest/api/rest_test.go | 5 ++- backend/app/rest/auth/auth_test.go | 9 ++-- backend/app/rest/auth/dev_provider_test.go | 5 ++- backend/app/rest/auth/jwt.go | 44 ++++++++++++++++--- backend/app/rest/auth/jwt_test.go | 19 ++++---- backend/app/rest/auth/provider.go | 2 +- backend/app/rest/auth/provider_test.go | 18 +++++--- backend/app/store/keys/keys.go | 27 ++++++++++++ backend/app/store/keys/keys_test.go | 20 +++++++++ backend/app/store/service/service.go | 12 ++++-- backend/app/store/service/service_test.go | 21 +++++---- 19 files changed, 217 insertions(+), 77 deletions(-) create mode 100644 backend/app/store/keys/keys.go create mode 100644 backend/app/store/keys/keys_test.go diff --git a/backend/app/main.go b/backend/app/main.go index a88b5759..483ed189 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/api" @@ -29,13 +30,14 @@ import ( // Opts with command line flags and env type Opts struct { - SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"` - RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` + RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` + SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` + Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` @@ -106,6 +108,11 @@ type MongoGroup struct { DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` } +// KeyGroup defines options group for key params +type KeyGroup struct { + Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` +} + var revision = "unknown" // Application holds all active objects @@ -166,10 +173,15 @@ func New(opts Opts) (*Application, error) { return nil, err } + keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret) + if err != nil { + return nil, err + } + dataService := &service.DataStore{ Interface: storeEngine, EditDuration: opts.EditDuration, - Secret: opts.SecretKey, + KeyStore: keyStore, MaxCommentSize: opts.MaxCommentSize, Admins: opts.Admins, } @@ -180,8 +192,7 @@ func New(opts Opts) (*Application, error) { } // token TTL is 5 minutes, inactivity interval 7+ days by default - jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), - opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) + jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) if err != nil { @@ -202,20 +213,21 @@ func New(opts Opts) (*Application, error) { DisqusImporter: &migrator.Disqus{DataStore: dataService}, WordPressImporter: &migrator.WordPress{DataStore: dataService}, NativeExported: &migrator.Remark{DataStore: dataService}, - SecretKey: opts.SecretKey, + KeyStore: keyStore, } authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) srv := &api.Rest{ - Version: revision, - DataService: dataService, - Exporter: exporter, - WebRoot: opts.WebRoot, - RemarkURL: opts.RemarkURL, - ImageProxy: &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}, - AvatarProxy: avatarProxy, - ReadOnlyAge: opts.ReadOnlyAge, + Version: revision, + DataService: dataService, + Exporter: exporter, + WebRoot: opts.WebRoot, + RemarkURL: opts.RemarkURL, + ImageProxy: &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}, + AvatarProxy: avatarProxy, + ReadOnlyAge: opts.ReadOnlyAge, + SharedSecret: opts.SharedSecret, Authenticator: auth.Authenticator{ JWTService: jwtService, AdminEmail: opts.AdminEmail, @@ -336,6 +348,15 @@ func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { return nil, errors.Errorf("unsupported avatar store type %s", group.Type) } +func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) { + switch group.Type { + case "shared": + return keys.NewStaticStore(sharedSecret), nil + default: + return nil, errors.Errorf("unsupported key store type %s", group.Type) + } +} + func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { switch group.Type { case "mem": @@ -398,7 +419,6 @@ func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *serv RemarkURL: opts.RemarkURL, Cid: cid, Csecret: secret, - SecretKey: opts.SecretKey, PermissionChecker: ds, } } diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index af7a55ba..a8e435e3 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" "github.com/stretchr/testify/assert" @@ -19,7 +20,7 @@ func TestDisqus_Import(t *testing.T) { defer os.Remove("/tmp/remark-test.db") b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := service.DataStore{Interface: b} + dataStore := service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} d := Disqus{DataStore: &dataStore} size, err := d.Import(strings.NewReader(xmlTestDisqus), "test") assert.Nil(t, err) @@ -36,7 +37,7 @@ func TestDisqus_Import(t *testing.T) { assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator) assert.Equal(t, "Dmitry Noname", c.User.Name) assert.Equal(t, "disqus_8799342cdf328253e03313958ffc6a433659d7ff", c.User.ID) - assert.Equal(t, "96243f024cf6ad42b66f0c72709ae20b5d10ec14", c.User.IP) + assert.Equal(t, "7001968ea3f6c9013a9f0a3650f200c10c927638", c.User.IP) posts, err := dataStore.List("test", 0, 0) assert.Nil(t, err) diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 120b1185..2ec70199 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -8,6 +8,7 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" "github.com/umputun/remark/backend/app/store/engine" @@ -24,7 +25,7 @@ func TestMigrator_ImportDisqus(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b} + dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} size, err := ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/disqus-test.xml", @@ -50,7 +51,7 @@ func TestMigrator_ImportWordPress(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b} + dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} size, err := ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/wordpress-test.xml", @@ -79,7 +80,7 @@ func TestMigrator_ImportRemark(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b} + dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} size, err := ImportComments(ImportParams{ DataStore: dataStore, diff --git a/backend/app/migrator/remark_test.go b/backend/app/migrator/remark_test.go index 414ec90f..572060d5 100644 --- a/backend/app/migrator/remark_test.go +++ b/backend/app/migrator/remark_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) @@ -49,7 +50,7 @@ func TestRemark_Import(t *testing.T) { os.Remove(testDb) b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - r := Remark{DataStore: &service.DataStore{Interface: b}} + r := Remark{DataStore: &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}} size, err := r.Import(buf, "radio-t") assert.Nil(t, err) assert.Equal(t, 2, size) @@ -77,7 +78,7 @@ func TestRemark_ImportManyWithError(t *testing.T) { os.Remove(testDb) b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - r := Remark{DataStore: &service.DataStore{Interface: b}} + r := Remark{DataStore: &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}} n, err := r.Import(buf, "radio-t") assert.EqualError(t, err, "failed to save 2 comments") assert.Equal(t, 1200, n) @@ -93,7 +94,7 @@ func prep(t *testing.T) *service.DataStore { boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - b := &service.DataStore{Interface: boltStore} + b := &service.DataStore{Interface: boltStore, KeyStore: keys.NewStaticStore("12345")} comment := store.Comment{ ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc", diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go index c915956a..d1cef135 100644 --- a/backend/app/migrator/wordpress_test.go +++ b/backend/app/migrator/wordpress_test.go @@ -8,6 +8,7 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" @@ -20,7 +21,7 @@ func TestWordPress_Import(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) assert.Nil(t, err, "create store") - dataStore := service.DataStore{Interface: b} + dataStore := service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} wp := WordPress{DataStore: &dataStore} size, err := wp.Import(strings.NewReader(xmlTestWP), siteID) assert.Nil(t, err) @@ -35,7 +36,7 @@ func TestWordPress_Import(t *testing.T) { assert.Equal(t, store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID}, c.Locator) assert.Equal(t, "wordpress_75b2b81081f82495d7af26759e67af6554ffda4a", c.User.ID) assert.Equal(t, "SuperUser3", c.User.Name) - assert.Equal(t, "b646e160768fbc1414d3b2c8f88a767bfbb00871", c.User.IP) + assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP) ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14") assert.Equal(t, ts, c.Timestamp) assert.Equal(t, c.Text, "Mekkatorque was over in that tent up to the right") diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index bb891bd2..3fdc3d9d 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -15,6 +15,7 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/go-chi/render" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" @@ -29,7 +30,7 @@ type Migrator struct { DisqusImporter migrator.Importer WordPressImporter migrator.Importer NativeExported migrator.Exporter - SecretKey string + KeyStore keys.Store httpServer *http.Server lock sync.Mutex @@ -81,15 +82,22 @@ func (m *Migrator) routes() chi.Router { // imports comments from post body. func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site") secret := r.URL.Query().Get("secret") - if strings.TrimSpace(secret) == "" || secret != m.SecretKey { + + skey, err := m.KeyStore.Get(siteID) + if err != nil { + render.Status(r, http.StatusForbidden) + render.JSON(w, r, JSON{"status": "error", "details": "secret key store"}) + return + } + + if strings.TrimSpace(secret) == "" || secret != skey { render.Status(r, http.StatusForbidden) render.JSON(w, r, JSON{"status": "error", "details": "secret key"}) return } - siteID := r.URL.Query().Get("site") - var importer migrator.Importer switch r.URL.Query().Get("provider") { case "disqus": @@ -116,14 +124,22 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { // exports all comments for siteID as gz file func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site") secret := r.URL.Query().Get("secret") - if strings.TrimSpace(secret) == "" || secret != m.SecretKey { + + skey, err := m.KeyStore.Get(siteID) + if err != nil { + render.Status(r, http.StatusForbidden) + render.JSON(w, r, JSON{"status": "error", "details": "secret key store"}) + return + } + + if strings.TrimSpace(secret) == "" || secret != skey { render.Status(r, http.StatusForbidden) render.JSON(w, r, JSON{"status": "error", "details": "secret key"}) return } - siteID := r.URL.Query().Get("site") exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102")) log.Printf("[DEBUG] import request for site=%s to %s", siteID, exportFile) diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 43f6c0e4..6de6342b 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -13,6 +13,7 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/cache" @@ -127,14 +128,14 @@ func TestMigrator_Shutdown(t *testing.T) { func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) - dataStore := &service.DataStore{Interface: b} + dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("123456")} svc = &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, NativeImporter: &migrator.Remark{DataStore: dataStore}, NativeExported: &migrator.Remark{DataStore: dataStore}, Cache: &cache.Nop{}, - SecretKey: "123456", + KeyStore: keys.NewStaticStore("123456"), } routes := svc.routes() @@ -142,7 +143,7 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) { return svc, ts } -func cleanupImportSrv(srv *Migrator, ts *httptest.Server) { +func cleanupImportSrv(_ *Migrator, ts *httptest.Server) { ts.Close() os.Remove(testDb) } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index ddef456d..c83db44c 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -43,6 +43,7 @@ type Rest struct { WebRoot string RemarkURL string ReadOnlyAge int + SharedSecret string ScoreThresholds struct { Low int Critical int @@ -129,7 +130,7 @@ func (s *Rest) routes() chi.Router { }) router.Use(corsMiddleware.Handler) - ipFn := func(ip string) string { return store.HashValue(ip, s.DataService.Secret)[:12] } // logger uses it for anonymization + ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization // auth routes for all providers router.Route("/auth", func(r chi.Router) { diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 936e8df2..9a691c76 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -14,6 +14,7 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/auth" @@ -78,7 +79,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { Interface: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, - Secret: "123456", + KeyStore: keys.NewStaticStore("123456"), Admins: []string{"a1", "a2"}, } srv = &Rest{ @@ -88,7 +89,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { Providers: nil, AdminEmail: "admin@remark-42.com", - JWTService: auth.NewJWT("12345", false, time.Minute, time.Hour), + JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), }, Exporter: &migrator.Remark{DataStore: dataStore}, Cache: &cache.Nop{}, diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index e7a50d71..33dc0327 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -11,6 +11,7 @@ import ( "github.com/go-chi/chi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/keys" ) var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOnRydWV9LCJzdGF0ZSI6IjEyMzQ1NiIsImZyb20iOiJmcm9tIn0.6P_OwGf8CUJRtvNSlW20GmaMb5pFvCNemP94fHCqb5Q" @@ -18,7 +19,7 @@ var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxO var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZmxhZ3MiOnsiZGVsZXRlbWUiOnRydWV9fQ.SLh1QpFytWZqcT99VgcdAOtgFKhvpKCcZwqWTvAd63g" func TestAuthJWTCookie(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour), + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour), PermissionChecker: &mockUserPermissions{}} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { @@ -56,7 +57,7 @@ func TestAuthJWTCookie(t *testing.T) { } func TestAuthJWTHeader(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour)} + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) @@ -83,7 +84,7 @@ func TestAuthJWTHeader(t *testing.T) { } func TestAuthJWtBlocked(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour)} + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) @@ -103,7 +104,7 @@ func TestAuthJWtBlocked(t *testing.T) { } func TestAuthJWtFlags(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour)} + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) diff --git a/backend/app/rest/auth/dev_provider_test.go b/backend/app/rest/auth/dev_provider_test.go index 448ac29c..8f768baa 100644 --- a/backend/app/rest/auth/dev_provider_test.go +++ b/backend/app/rest/auth/dev_provider_test.go @@ -13,11 +13,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/keys" ) func TestDevProvider(t *testing.T) { - params := Params{RemarkURL: "http://127.0.0.1:8080", SecretKey: "123456", Cid: "cid", Csecret: "csecret", - JwtService: NewJWT("12345", false, time.Hour, time.Hour*24*31), + params := Params{RemarkURL: "http://127.0.0.1:8080", Cid: "cid", Csecret: "csecret", + JwtService: NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31), PermissionChecker: &mockUserPermissions{admin: "dev_user"}, } srv := DevAuthServer{Provider: NewDev(params), nonInteractive: true, username: "dev_user"} diff --git a/backend/app/rest/auth/jwt.go b/backend/app/rest/auth/jwt.go index 54855432..0269ad0f 100644 --- a/backend/app/rest/auth/jwt.go +++ b/backend/app/rest/auth/jwt.go @@ -4,8 +4,9 @@ import ( "net/http" "time" - jwt "github.com/dgrijalva/jwt-go" + "github.com/dgrijalva/jwt-go" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" ) @@ -13,7 +14,7 @@ import ( // JWT wraps jwt operations // supports both header and cookie jwt type JWT struct { - secret string + keyStore keys.Store secureCookies bool tokenDuration time.Duration cookieDuration time.Duration @@ -43,9 +44,9 @@ const xsrfCookieName = "XSRF-TOKEN" const xsrfHeaderKey = "X-XSRF-TOKEN" // NewJWT makes JWT service -func NewJWT(secret string, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT { +func NewJWT(keyStore keys.Store, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT { res := JWT{ - secret: secret, + keyStore: keyStore, secureCookies: secureCookies, tokenDuration: tokenDuration, cookieDuration: cookieDuration, @@ -56,14 +57,20 @@ func NewJWT(secret string, secureCookies bool, tokenDuration time.Duration, cook // Token makes jwt with claims func (j *JWT) Token(claims *CustomClaims) (string, error) { token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - tokenString, err := token.SignedString([]byte(j.secret)) + + secret, err := j.keyStore.Get(claims.SiteID) + if err != nil { + return "", errors.Wrap(err, "can't get secret") + } + + tokenString, err := token.SignedString([]byte(secret)) if err != nil { return "", errors.Wrap(err, "can't sign jwt token") } return tokenString, nil } -// HasFlags indicates presense of special flags +// HasFlags indicates presence of special flags func (j *JWT) HasFlags(claims *CustomClaims) bool { return claims.Flags.DeleteMe || claims.Flags.Login } @@ -71,11 +78,34 @@ func (j *JWT) HasFlags(claims *CustomClaims) bool { // Parse token string and verify. Not checking for expiration func (j *JWT) Parse(tokenString string) (*CustomClaims, error) { parser := jwt.Parser{SkipClaimsValidation: true} // allow parsing of expired tokens + + getSiteID := func() (siteID string, err error) { // parse token without signature check to get siteID + preToken, _, err := parser.ParseUnverified(tokenString, &CustomClaims{}) + if err != nil { + return "", errors.Wrap(err, "can't pre-parse jwt") + } + preClaims, ok := preToken.Claims.(*CustomClaims) + if !ok { + return "", errors.New("invalid jwt") + } + return preClaims.SiteID, nil + } + + siteID, err := getSiteID() + if err != nil { + return nil, errors.Wrap(err, "failed to get siteID from jwt token") + } + + secret, err := j.keyStore.Get(siteID) + if err != nil { + return nil, errors.Wrap(err, "can't get secret") + } + token, err := parser.ParseWithClaims(tokenString, &CustomClaims{}, func(token *jwt.Token) (interface{}, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { return nil, errors.Errorf("unexpected signing method: %v", token.Header["alg"]) } - return []byte(j.secret), nil + return []byte(secret), nil }) if err != nil { return nil, errors.Wrap(err, "can't parse jwt") diff --git a/backend/app/rest/auth/jwt_test.go b/backend/app/rest/auth/jwt_test.go index 0795809b..f1ce7efa 100644 --- a/backend/app/rest/auth/jwt_test.go +++ b/backend/app/rest/auth/jwt_test.go @@ -10,6 +10,7 @@ import ( "github.com/dgrijalva/jwt-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" ) @@ -29,7 +30,7 @@ var testJwtBadSign = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4M var days31 = time.Hour * 24 * 31 func TestJWT_Token(t *testing.T) { - j := NewJWT("xyz 12345", false, time.Hour, days31) + j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -52,7 +53,7 @@ func TestJWT_Token(t *testing.T) { } func TestJWT_Parse(t *testing.T) { - j := NewJWT("xyz 12345", false, time.Hour, days31) + j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) claims, err := j.Parse(testJwtValid) assert.NoError(t, err) assert.False(t, j.IsExpired(claims)) @@ -70,7 +71,7 @@ func TestJWT_Parse(t *testing.T) { } func TestJWT_Set(t *testing.T) { - j := NewJWT("xyz 12345", false, time.Hour, days31) + j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -115,7 +116,7 @@ func TestJWT_Set(t *testing.T) { } func TestJWT_GetFromHeader(t *testing.T) { - j := NewJWT("xyz 12345", false, time.Hour, days31) + j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) req := httptest.NewRequest("GET", "/", nil) req.Header.Add(jwtHeaderKey, testJwtValid) @@ -134,13 +135,13 @@ func TestJWT_GetFromHeader(t *testing.T) { req = httptest.NewRequest("GET", "/", nil) req.Header.Add(jwtHeaderKey, "bad bad token") _, err = j.Get(req) - assert.NotNil(t, err) - assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token contains an invalid number of segments"), err.Error()) + require.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "can't pre-parse jwt: token contains an invalid number of segments"), err.Error()) } func TestJWT_SetAndGetWithCookies(t *testing.T) { - j := NewJWT("xyz 12345", false, time.Hour, days31) + j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -182,7 +183,7 @@ func TestJWT_SetAndGetWithCookies(t *testing.T) { } func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) { - j := NewJWT("xyz 12345", false, time.Hour, days31) + j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -219,7 +220,7 @@ func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) { } func TestJWT_SetAndGetWithCookiesExpired(t *testing.T) { - j := NewJWT("xyz 12345", false, time.Hour, days31) + j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", diff --git a/backend/app/rest/auth/provider.go b/backend/app/rest/auth/provider.go index 30f87739..1fe0588d 100644 --- a/backend/app/rest/auth/provider.go +++ b/backend/app/rest/auth/provider.go @@ -39,7 +39,6 @@ type Params struct { AvatarProxy *proxy.Avatar JwtService *JWT PermissionChecker PermissionChecker - SecretKey string Cid string Csecret string } @@ -171,6 +170,7 @@ func (p Provider) authHandler(w http.ResponseWriter, r *http.Request) { Issuer: "remark42", Id: p.randToken(), }, + SiteID: oauthClaims.SiteID, SessionOnly: oauthClaims.SessionOnly, } diff --git a/backend/app/rest/auth/provider_test.go b/backend/app/rest/auth/provider_test.go index 14e83a8f..c26c9087 100644 --- a/backend/app/rest/auth/provider_test.go +++ b/backend/app/rest/auth/provider_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/keys" "golang.org/x/oauth2" "github.com/umputun/remark/backend/app/store" @@ -52,6 +53,14 @@ func TestLogin(t *testing.T) { assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser1", Picture: "http://exmple.com/pic1.png", Admin: false, Blocked: true, IP: ""}, u) + token := resp.Cookies()[0].Value + jwtSvc := NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31) + + claims, err := jwtSvc.Parse(token) + require.NoError(t, err) + assert.Equal(t, "remark42", claims.Issuer) + assert.Equal(t, "remark", claims.SiteID) + // check admin user resp, err = client.Get("http://localhost:8981/login?site=remark") assert.Nil(t, err) @@ -94,7 +103,7 @@ func TestLoginSessionOnly(t *testing.T) { req.AddCookie(resp.Cookies()[1]) req.Header.Add("X-XSRF-TOKEN", resp.Cookies()[1].Value) - jwtService := NewJWT("12345", false, time.Hour, time.Hour) + jwtService := NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour) res, err := jwtService.Get(req) require.Nil(t, err) assert.Equal(t, true, res.SessionOnly) @@ -129,13 +138,12 @@ func TestLogout(t *testing.T) { } func TestInitProvider(t *testing.T) { - params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret"} + params := Params{RemarkURL: "url", Cid: "cid", Csecret: "csecret"} provider := Provider{Name: "test", RedirectURL: "redir"} res := initProvider(params, provider) assert.Equal(t, "cid", res.conf.ClientID) assert.Equal(t, "csecret", res.conf.ClientSecret) assert.Equal(t, "redir", res.RedirectURL) - assert.Equal(t, "123456", res.SecretKey) assert.Equal(t, "test", res.Name) } @@ -160,8 +168,8 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se }, } - params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret", - JwtService: NewJWT("12345", false, time.Hour, time.Hour*24*31), + params := Params{RemarkURL: "url", Cid: "cid", Csecret: "csecret", + JwtService: NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31), // AvatarProxy: &proxy.Avatar{Store: &mockAvatarStore, RoutePath: "/v1/avatar"}, PermissionChecker: &mockUserPermissions{admin: "mock_myuser2", verified: "mock_myuser2", blocked: "mock_myuser1"}, } diff --git a/backend/app/store/keys/keys.go b/backend/app/store/keys/keys.go new file mode 100644 index 00000000..36353a6c --- /dev/null +++ b/backend/app/store/keys/keys.go @@ -0,0 +1,27 @@ +package keys + +import "github.com/pkg/errors" + +// Store defines interface returning key for given site +// this key used for JWT and HMAC hashes +type Store interface { + Get(siteID string) (key string, err error) +} + +// StaticStore implements keys.Store with a single, predefined key +type StaticStore struct { + key string +} + +// NewStaticStore makes StaticStore instance with given key +func NewStaticStore(key string) *StaticStore { + return &StaticStore{key: key} +} + +// Get returns static key for all sites, allows empty site +func (s *StaticStore) Get(siteID string) (key string, err error) { + if s.key == "" { + return "", errors.New("empty key for static key store") + } + return s.key, nil +} diff --git a/backend/app/store/keys/keys_test.go b/backend/app/store/keys/keys_test.go new file mode 100644 index 00000000..c4b47eb2 --- /dev/null +++ b/backend/app/store/keys/keys_test.go @@ -0,0 +1,20 @@ +package keys + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStaticStore_Get(t *testing.T) { + var ks Store = NewStaticStore("key123") + + k, err := ks.Get("any") + assert.NoError(t, err, "valid store") + assert.Equal(t, "key123", k, "valid site") + + ks = NewStaticStore("") + + _, err = ks.Get("any") + assert.NotNil(t, err, "invalid (empty key) store") +} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index c7fd97ad..d26d898b 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" @@ -15,7 +16,7 @@ import ( type DataStore struct { engine.Interface EditDuration time.Duration - Secret string + KeyStore keys.Store MaxCommentSize int Admins []string @@ -43,8 +44,13 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) comment.Votes = make(map[string]bool) } - comment.Sanitize() // clear potentially dangerous js from all parts of comment - comment.User.HashIP(s.Secret) // replace ip by hash + comment.Sanitize() // clear potentially dangerous js from all parts of comment + + secret, err := s.KeyStore.Get(comment.Locator.SiteID) + if err != nil { + return "", errors.Wrapf(err, "can't get secret for site %s", comment.Locator.SiteID) + } + comment.User.HashIP(secret) // replace ip by hash return s.Interface.Create(comment) } diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index f5a76f19..590a65ba 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" @@ -22,7 +23,8 @@ var testDb = "/tmp/test-remark.db" func TestService_CreateFromEmpty(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"} + ks := keys.NewStaticStore("secret 123") + b := DataStore{Interface: prepStoreEngine(t), KeyStore: ks} comment := store.Comment{ Text: "text", User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"}, @@ -45,7 +47,8 @@ func TestService_CreateFromEmpty(t *testing.T) { func TestService_CreateFromPartial(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"} + ks := keys.NewStaticStore("secret 123") + b := DataStore{Interface: prepStoreEngine(t), KeyStore: ks} comment := store.Comment{ Text: "text", Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC), @@ -70,7 +73,7 @@ func TestService_CreateFromPartial(t *testing.T) { func TestService_Vote(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t)} + b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} comment := store.Comment{ Text: "text", @@ -115,7 +118,7 @@ func TestService_Vote(t *testing.T) { func TestService_VoteAggressive(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t)} + b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} comment := store.Comment{ Text: "text", @@ -175,7 +178,7 @@ func TestService_VoteAggressive(t *testing.T) { func TestService_VoteConcurrent(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t)} + b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} comment := store.Comment{ Text: "text", @@ -206,7 +209,7 @@ func TestService_VoteConcurrent(t *testing.T) { func TestService_Pin(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t)} + b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -230,7 +233,7 @@ func TestService_Pin(t *testing.T) { func TestService_EditComment(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t)} + b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -257,7 +260,7 @@ func TestService_EditComment(t *testing.T) { func TestService_EditCommentDurationFailed(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond} + b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, KeyStore: keys.NewStaticStore("secret 123")} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -274,7 +277,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) { func TestService_ValidateComment(t *testing.T) { - b := DataStore{MaxCommentSize: 2000} + b := DataStore{MaxCommentSize: 2000, KeyStore: keys.NewStaticStore("secret 123")} longText := fmt.Sprintf("%4000s", "X") tbl := []struct { From 08e902e785955ea319ad5753360b71932a754771 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 9 Aug 2018 01:57:18 -0500 Subject: [PATCH 44/93] add table of contents --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2eb7abf0..cc8a25b4 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,44 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * Integration with automatic ssl via [nginx-le](https://github.com/umputun/nginx-le) * [Privacy focused](#privacy) +---- + +- [remark42 ![Build Status](https://travis-ci.org/umputun/remark) ![Go Report Card](https://goreportcard.com/report/github.com/umputun/remark) ![Coverage Status](https://coveralls.io/github/umputun/remark?branch=master)](#remark42-build-statushttpstravis-ciorgumputunremark-go-report-cardhttpsgoreportcardcomreportgithubcomumputunremark-coverage-statushttpscoverallsiogithubumputunremarkbranchmaster) + - [Install](#install) + - [Backend](#backend) + - [Parameters](#parameters) + - [Required parameters](#required-parameters) + - [Register oauth2 providers](#register-oauth2-providers) + - [Google Auth Provider](#google-auth-provider) + - [GitHub Auth Provider](#github-auth-provider) + - [Facebook Auth Provider](#facebook-auth-provider) + - [Yandex Auth Provider](#yandex-auth-provider) + - [Initial import from Disqus](#initial-import-from-disqus) + - [Initial import from WordPress](#initial-import-from-wordpress) + - [Backup and restore](#backup-and-restore) + - [Automatic backups](#automatic-backups) + - [Schema migration](#schema-migration) + - [Manual backup](#manual-backup) + - [Backup format](#backup-format) + - [Admin users](#admin-users) + - [Setup on your website](#setup-on-your-website) + - [Comments](#comments) + - [Last comments](#last-comments) + - [Counter](#counter) + - [Development](#development) + - [Backend development](#backend-development) + - [Frontend development](#frontend-development) + - [Build](#build) + - [Devserver](#devserver) + - [API](#api) + - [Authorization](#authorization) + - [Commenting](#commenting) + - [RSS feeds](#rss-feeds) + - [Admin](#admin) + - [Privacy](#privacy) + - [Technical details](#technical-details) +---- + ## Install ### Backend @@ -240,7 +278,7 @@ And then add this node in the place where you want to see Remark42 widget: After that widget will be rendered inside this node. -##### Last comments +#### Last comments It's a widget which renders list of last comments from your site. @@ -268,7 +306,7 @@ And then add this node in the place where you want to see last comments widget: `data-max` sets the max amount of comments (default: `15`). -##### Counter +#### Counter It's a widget which renders a number of comments for the specified page. @@ -323,7 +361,7 @@ You can tweak any of [supported parameters](#Parameters) in corresponded yml fil Backend docker compose config by default skips running frontend related tests. Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build. -### Backend +### Backend development In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install). @@ -335,7 +373,7 @@ It stars backend service with embedded bolt store on port `8080` with basic auth To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost` -### Frontend +### Frontend development #### Build From f15af0c7dc069292a3f3e1751de6b7f2842326d9 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 9 Aug 2018 01:58:30 -0500 Subject: [PATCH 45/93] fix table --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index cc8a25b4..dfaf6f04 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi ---- -- [remark42 ![Build Status](https://travis-ci.org/umputun/remark) ![Go Report Card](https://goreportcard.com/report/github.com/umputun/remark) ![Coverage Status](https://coveralls.io/github/umputun/remark?branch=master)](#remark42-build-statushttpstravis-ciorgumputunremark-go-report-cardhttpsgoreportcardcomreportgithubcomumputunremark-coverage-statushttpscoverallsiogithubumputunremarkbranchmaster) - [Install](#install) - [Backend](#backend) - [Parameters](#parameters) From 2ff955edf150b13d8c175647167e9a7f8708be31 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 10 Aug 2018 19:35:45 -0500 Subject: [PATCH 46/93] feature/comment-formater (#186) * extract comment formatter functuionality to allow md in #156 * add abilty to pass multiple convertors to formater --- backend/app/main.go | 22 ++++--- backend/app/rest/api/rest.go | 10 ++- backend/app/rest/api/rest_private.go | 9 +-- backend/app/rest/api/rest_public.go | 6 +- backend/app/rest/api/rest_test.go | 15 +++-- backend/app/store/comment.go | 38 ----------- backend/app/store/comment_test.go | 55 ++-------------- backend/app/store/fomrater.go | 79 ++++++++++++++++++++++ backend/app/store/formater_test.go | 98 ++++++++++++++++++++++++++++ backend/app/store/service/service.go | 1 - 10 files changed, 212 insertions(+), 121 deletions(-) create mode 100644 backend/app/store/fomrater.go create mode 100644 backend/app/store/formater_test.go diff --git a/backend/app/main.go b/backend/app/main.go index 483ed189..acc7a748 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" @@ -217,17 +218,20 @@ func New(opts Opts) (*Application, error) { } authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) + imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} + commentFormater := store.NewCommentFormater(imgProxy) srv := &api.Rest{ - Version: revision, - DataService: dataService, - Exporter: exporter, - WebRoot: opts.WebRoot, - RemarkURL: opts.RemarkURL, - ImageProxy: &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}, - AvatarProxy: avatarProxy, - ReadOnlyAge: opts.ReadOnlyAge, - SharedSecret: opts.SharedSecret, + Version: revision, + DataService: dataService, + Exporter: exporter, + WebRoot: opts.WebRoot, + RemarkURL: opts.RemarkURL, + ImageProxy: imgProxy, + CommentFormater: commentFormater, + AvatarProxy: avatarProxy, + ReadOnlyAge: opts.ReadOnlyAge, + SharedSecret: opts.SharedSecret, Authenticator: auth.Authenticator{ JWTService: jwtService, AdminEmail: opts.AdminEmail, diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index c83db44c..7746e936 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -20,7 +20,6 @@ import ( "github.com/go-chi/cors" "github.com/go-chi/render" "github.com/pkg/errors" - "gopkg.in/russross/blackfriday.v2" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" @@ -33,13 +32,16 @@ import ( // Rest is a rest access server type Rest struct { - Version string + Version string + DataService *service.DataStore Authenticator auth.Authenticator Exporter migrator.Exporter Cache cache.LoadingCache AvatarProxy *proxy.Avatar ImageProxy *proxy.Image + CommentFormater *store.CommentFormater + WebRoot string RemarkURL string ReadOnlyAge int @@ -57,10 +59,6 @@ type Rest struct { const hardBodyLimit = 1024 * 64 // limit size of body -var mdExt = blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode | - blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak | - blackfriday.BackslashLineBreak | blackfriday.Autolink - type commentsWithInfo struct { Comments []store.Comment `json:"comments"` Info store.PostInfo `json:"info,omitempty"` diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 62a2163a..b8250593 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -14,7 +14,6 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" "github.com/hashicorp/go-multierror" - "gopkg.in/russross/blackfriday.v2" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/auth" @@ -44,8 +43,8 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment") return } - comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt))) - comment.Text = s.ImageProxy.Convert(comment.Text) + comment = s.CommentFormater.Format(comment) + // check if user blocked if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) { rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked") @@ -109,10 +108,8 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - text := string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))) // render markdown - text = s.ImageProxy.Convert(text) editReq := service.EditRequest{ - Text: text, + Text: s.CommentFormater.FormatText(edit.Text), Orig: edit.Text, Summary: edit.Summary, } diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index bf397ab0..5315ebc0 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -10,7 +10,6 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" - "gopkg.in/russross/blackfriday.v2" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/cache" @@ -79,10 +78,7 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - //comment.Text = string(blackfriday.Run([]byte(comment.Text), - // blackfriday.WithRenderer(bfchroma.NewRenderer(bfchroma.WithoutAutodetect())))) - comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt))) - comment.Text = s.ImageProxy.Convert(comment.Text) + comment = s.CommentFormater.Format(comment) comment.Sanitize() render.HTML(w, r, comment.Text) } diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 9a691c76..218dfb05 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -91,13 +91,14 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { AdminEmail: "admin@remark-42.com", JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), }, - Exporter: &migrator.Remark{DataStore: dataStore}, - Cache: &cache.Nop{}, - WebRoot: "/tmp", - RemarkURL: "https://demo.remark42.com", - AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"}, - ImageProxy: &proxy.Image{}, - ReadOnlyAge: 10, + Exporter: &migrator.Remark{DataStore: dataStore}, + Cache: &cache.Nop{}, + WebRoot: "/tmp", + RemarkURL: "https://demo.remark42.com", + AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"}, + ImageProxy: &proxy.Image{}, + ReadOnlyAge: 10, + CommentFormater: store.NewCommentFormater(&proxy.Image{}), } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10 diff --git a/backend/app/store/comment.go b/backend/app/store/comment.go index 0fe916bb..51c5fb38 100644 --- a/backend/app/store/comment.go +++ b/backend/app/store/comment.go @@ -2,12 +2,9 @@ package store import ( "html/template" - "net/url" "regexp" - "strings" "time" - "github.com/PuerkitoBio/goquery" "github.com/microcosm-cc/bluemonday" ) @@ -102,43 +99,8 @@ func (c *Comment) Sanitize() { p := bluemonday.UGCPolicy() p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code") c.Text = p.Sanitize(c.Text) - c.Text = shortenAutoLinks(c.Text, shortURLLen) c.Orig = p.Sanitize(c.Orig) c.User.ID = template.HTMLEscapeString(c.User.ID) c.User.Name = template.HTMLEscapeString(c.User.Name) c.User.Picture = p.Sanitize(c.User.Picture) } - -// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes. -func shortenAutoLinks(commentHTML string, max int) (resHTML string) { - doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML)) - if err != nil { - return commentHTML - } - doc.Find("a").Each(func(i int, s *goquery.Selection) { - if href, ok := s.Attr("href"); ok { - if href != s.Text() || len(href) < max+3 || max < 3 { - return - } - url, e := url.Parse(href) - if e != nil { - return - } - url.Path, url.RawQuery, url.Fragment = "", "", "" - host := url.String() - if host == "" { - return - } - short := href[:max-3] - if len(short) < len(host) { - short = host - } - s.SetText(short + "...") - } - }) - resHTML, err = doc.Find("body").Html() - if err != nil { - return commentHTML - } - return resHTML -} diff --git a/backend/app/store/comment_test.go b/backend/app/store/comment_test.go index 1ce03754..2829382b 100644 --- a/backend/app/store/comment_test.go +++ b/backend/app/store/comment_test.go @@ -17,21 +17,21 @@ func TestComment_Sanitize(t *testing.T) { { inp: Comment{ Text: `blah XSS` + "\n\t", - User: User{ID: `username`}, + User: User{ID: `username`, Name: "name "}, }, out: Comment{ Text: "blah XSS\n\t", - User: User{ID: `<a href="http://blah.com">username</a>`}, + User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name <b/>"}, }, }, { inp: Comment{ - Text: `blah https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/` + "\n\t", - User: User{ID: `username`}, + Text: "blah 123" + "\n\t", + User: User{ID: "id", Name: "xyz"}, }, out: Comment{ - Text: `blah https://www.reddit.com/r/golang/comments/8jdo...` + "\n\t", - User: User{ID: `<a href="http://blah.com">username</a>`}, + Text: `blah 123` + "\n\t", + User: User{ID: "id", Name: "xyz"}, }, }, } @@ -120,46 +120,3 @@ func TestComment_SetDeletedHard(t *testing.T) { assert.False(t, comment.Pin) assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User) } - -func TestComment_ShortenAutoLinks(t *testing.T) { - tbl := []struct { - max int - in, out string - }{ - {32, "", ""}, - {32, "text", "text"}, - {32, "

asd

", "

asd

"}, - {5, `incorrect-url`, `incorrect-url`}, - {32, `some text, not href`, `some text, not href`}, - { - 32, - `https://blah.com/a/b/c/d?g=123#anc`, - `https://blah.com/a/b/c/d?g=123#anc`, - }, - { - 31, - `https://blah.com/a/b/c/d?g=123#anc`, - `https://blah.com/a/b/c/d?g=1...`, - }, - { - 15, - `https://blah.com/a/b/c/d?g=123#anc`, - `https://blah.com...`, - }, - { - 3, - `https://blah.com/a/b/c/d?g=123#anc`, - `https://blah.com...`, - }, - { - -1, - `https://blah.com/a/b/c/d?g=123#anc`, - `https://blah.com/a/b/c/d?g=123#anc`, - }, - } - - for n, tt := range tbl { - got := shortenAutoLinks(tt.in, tt.max) - assert.Equalf(t, tt.out, got, "check #%d", n) - } -} diff --git a/backend/app/store/fomrater.go b/backend/app/store/fomrater.go new file mode 100644 index 00000000..2154d4f3 --- /dev/null +++ b/backend/app/store/fomrater.go @@ -0,0 +1,79 @@ +package store + +import ( + "net/url" + "strings" + + "github.com/PuerkitoBio/goquery" + blackfriday "gopkg.in/russross/blackfriday.v2" +) + +// CommentFormater implements all generic formatings ops on comment +type CommentFormater struct { + converters []CommentConverter +} + +// CommentConverter defines interface to convert some parts of commentHTML +// Passed at creation time and does client-defined convertions, like image proxy link change +type CommentConverter interface { + Convert(text string) string +} + +// NewCommentFormater makes CommentFormater +func NewCommentFormater(converters ...CommentConverter) *CommentFormater { + return &CommentFormater{converters: converters} +} + +// Format comment fields +func (f *CommentFormater) Format(c Comment) Comment { + c.Text = f.FormatText(c.Text) + return c +} + +// FormatText formatting line +func (f *CommentFormater) FormatText(txt string) (res string) { + mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode | + blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak | + blackfriday.BackslashLineBreak | blackfriday.Autolink + res = string(blackfriday.Run([]byte(txt), blackfriday.WithExtensions(mdExt))) + for _, conv := range f.converters { + res = conv.Convert(res) + + } + res = f.shortenAutoLinks(res, shortURLLen) + return res +} + +// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes. +func (f *CommentFormater) shortenAutoLinks(commentHTML string, max int) (resHTML string) { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML)) + if err != nil { + return commentHTML + } + doc.Find("a").Each(func(i int, s *goquery.Selection) { + if href, ok := s.Attr("href"); ok { + if href != s.Text() || len(href) < max+3 || max < 3 { + return + } + url, e := url.Parse(href) + if e != nil { + return + } + url.Path, url.RawQuery, url.Fragment = "", "", "" + host := url.String() + if host == "" { + return + } + short := href[:max-3] + if len(short) < len(host) { + short = host + } + s.SetText(short + "...") + } + }) + resHTML, err = doc.Find("body").Html() + if err != nil { + return commentHTML + } + return resHTML +} diff --git a/backend/app/store/formater_test.go b/backend/app/store/formater_test.go new file mode 100644 index 00000000..cd33f24a --- /dev/null +++ b/backend/app/store/formater_test.go @@ -0,0 +1,98 @@ +package store + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type mockConvertor struct{} + +func (m mockConvertor) Convert(text string) string { return text + "!converted" } + +func TestFormater_FormatText(t *testing.T) { + tbl := []struct { + in, out string + }{ + {"", "!converted"}, + {"12345 abc", "

12345 abc

\n!converted"}, + {"**xyz** _aaa_", "

xyz aaa

\n!converted"}, + { + "http://127.0.0.1/some-long-link/12345/678901234567890", "

http://127.0.0.1/some-long-link/12345/6789012...

\n!converted", + }, + } + f := NewCommentFormater(mockConvertor{}) + for n, tt := range tbl { + assert.Equal(t, tt.out, f.FormatText(tt.in), "check #%d", n) + } +} + +func TestFormater_FormatTextNoConvertor(t *testing.T) { + f := NewCommentFormater() + assert.Equal(t, "

12345

\n", f.FormatText("12345")) +} + +func TestFormater_FormatComment(t *testing.T) { + comment := Comment{ + Text: `blah`, + User: User{ID: "username"}, + ParentID: "p123", + ID: "123", + Locator: Locator{SiteID: "site", URL: "url"}, + Score: 10, + Pin: true, + Deleted: true, + Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local), + Votes: map[string]bool{"uu": true}, + } + + f := NewCommentFormater(mockConvertor{}) + exp := comment + exp.Text = "

blah

\n!converted" + assert.Equal(t, exp, f.Format(comment)) +} + +func TestFormater_ShortenAutoLinks(t *testing.T) { + f := NewCommentFormater(nil) + tbl := []struct { + max int + in, out string + }{ + {32, "", ""}, + {32, "text", "text"}, + {32, "

asd

", "

asd

"}, + {5, `incorrect-url`, `incorrect-url`}, + {32, `some text, not href`, `some text, not href`}, + { + 32, + `https://blah.com/a/b/c/d?g=123#anc`, + `https://blah.com/a/b/c/d?g=123#anc`, + }, + { + 31, + `https://blah.com/a/b/c/d?g=123#anc`, + `https://blah.com/a/b/c/d?g=1...`, + }, + { + 15, + `https://blah.com/a/b/c/d?g=123#anc`, + `https://blah.com...`, + }, + { + 3, + `https://blah.com/a/b/c/d?g=123#anc`, + `https://blah.com...`, + }, + { + -1, + `https://blah.com/a/b/c/d?g=123#anc`, + `https://blah.com/a/b/c/d?g=123#anc`, + }, + } + + for n, tt := range tbl { + got := f.shortenAutoLinks(tt.in, tt.max) + assert.Equalf(t, tt.out, got, "check #%d", n) + } +} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index d26d898b..98f05fcf 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -43,7 +43,6 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) if comment.Votes == nil { comment.Votes = make(map[string]bool) } - comment.Sanitize() // clear potentially dangerous js from all parts of comment secret, err := s.KeyStore.Get(comment.Locator.SiteID) From c459b31aa816a9a4627121c2c1cc3737ecbb5afa Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 11 Aug 2018 11:29:31 -0500 Subject: [PATCH 47/93] fix typos in struct names --- backend/app/main.go | 22 +++++++++++----------- backend/app/rest/api/rest.go | 14 +++++++------- backend/app/rest/api/rest_private.go | 4 ++-- backend/app/rest/api/rest_public.go | 2 +- backend/app/rest/api/rest_test.go | 16 ++++++++-------- backend/app/store/fomrater.go | 24 ++++++++++++------------ backend/app/store/formater_test.go | 20 ++++++++++---------- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/backend/app/main.go b/backend/app/main.go index acc7a748..c15ba681 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -219,19 +219,19 @@ func New(opts Opts) (*Application, error) { authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} - commentFormater := store.NewCommentFormater(imgProxy) + commentFormatter := store.NewCommentFormatter(imgProxy) srv := &api.Rest{ - Version: revision, - DataService: dataService, - Exporter: exporter, - WebRoot: opts.WebRoot, - RemarkURL: opts.RemarkURL, - ImageProxy: imgProxy, - CommentFormater: commentFormater, - AvatarProxy: avatarProxy, - ReadOnlyAge: opts.ReadOnlyAge, - SharedSecret: opts.SharedSecret, + Version: revision, + DataService: dataService, + Exporter: exporter, + WebRoot: opts.WebRoot, + RemarkURL: opts.RemarkURL, + ImageProxy: imgProxy, + CommentFormatter: commentFormatter, + AvatarProxy: avatarProxy, + ReadOnlyAge: opts.ReadOnlyAge, + SharedSecret: opts.SharedSecret, Authenticator: auth.Authenticator{ JWTService: jwtService, AdminEmail: opts.AdminEmail, diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 7746e936..0a4b71cf 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -34,13 +34,13 @@ import ( type Rest struct { Version string - DataService *service.DataStore - Authenticator auth.Authenticator - Exporter migrator.Exporter - Cache cache.LoadingCache - AvatarProxy *proxy.Avatar - ImageProxy *proxy.Image - CommentFormater *store.CommentFormater + DataService *service.DataStore + Authenticator auth.Authenticator + Exporter migrator.Exporter + Cache cache.LoadingCache + AvatarProxy *proxy.Avatar + ImageProxy *proxy.Image + CommentFormatter *store.CommentFormatter WebRoot string RemarkURL string diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index b8250593..6f1edd0a 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -43,7 +43,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment") return } - comment = s.CommentFormater.Format(comment) + comment = s.CommentFormatter.Format(comment) // check if user blocked if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) { @@ -109,7 +109,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { } editReq := service.EditRequest{ - Text: s.CommentFormater.FormatText(edit.Text), + Text: s.CommentFormatter.FormatText(edit.Text), Orig: edit.Text, Summary: edit.Summary, } diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index 5315ebc0..6ee667bd 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -78,7 +78,7 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - comment = s.CommentFormater.Format(comment) + comment = s.CommentFormatter.Format(comment) comment.Sanitize() render.HTML(w, r, comment.Text) } diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 218dfb05..43a2f76c 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -91,14 +91,14 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { AdminEmail: "admin@remark-42.com", JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), }, - Exporter: &migrator.Remark{DataStore: dataStore}, - Cache: &cache.Nop{}, - WebRoot: "/tmp", - RemarkURL: "https://demo.remark42.com", - AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"}, - ImageProxy: &proxy.Image{}, - ReadOnlyAge: 10, - CommentFormater: store.NewCommentFormater(&proxy.Image{}), + Exporter: &migrator.Remark{DataStore: dataStore}, + Cache: &cache.Nop{}, + WebRoot: "/tmp", + RemarkURL: "https://demo.remark42.com", + AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"}, + ImageProxy: &proxy.Image{}, + ReadOnlyAge: 10, + CommentFormatter: store.NewCommentFormatter(&proxy.Image{}), } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10 diff --git a/backend/app/store/fomrater.go b/backend/app/store/fomrater.go index 2154d4f3..e51d7192 100644 --- a/backend/app/store/fomrater.go +++ b/backend/app/store/fomrater.go @@ -8,30 +8,30 @@ import ( blackfriday "gopkg.in/russross/blackfriday.v2" ) -// CommentFormater implements all generic formatings ops on comment -type CommentFormater struct { +// CommentFormatter implements all generic formatting ops on comment +type CommentFormatter struct { converters []CommentConverter } // CommentConverter defines interface to convert some parts of commentHTML -// Passed at creation time and does client-defined convertions, like image proxy link change +// Passed at creation time and does client-defined conversions, like image proxy link change type CommentConverter interface { Convert(text string) string } -// NewCommentFormater makes CommentFormater -func NewCommentFormater(converters ...CommentConverter) *CommentFormater { - return &CommentFormater{converters: converters} +// NewCommentFormatter makes CommentFormatter +func NewCommentFormatter(converters ...CommentConverter) *CommentFormatter { + return &CommentFormatter{converters: converters} } // Format comment fields -func (f *CommentFormater) Format(c Comment) Comment { +func (f *CommentFormatter) Format(c Comment) Comment { c.Text = f.FormatText(c.Text) return c } // FormatText formatting line -func (f *CommentFormater) FormatText(txt string) (res string) { +func (f *CommentFormatter) FormatText(txt string) (res string) { mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode | blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak | blackfriday.BackslashLineBreak | blackfriday.Autolink @@ -45,7 +45,7 @@ func (f *CommentFormater) FormatText(txt string) (res string) { } // Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes. -func (f *CommentFormater) shortenAutoLinks(commentHTML string, max int) (resHTML string) { +func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTML string) { doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML)) if err != nil { return commentHTML @@ -55,12 +55,12 @@ func (f *CommentFormater) shortenAutoLinks(commentHTML string, max int) (resHTML if href != s.Text() || len(href) < max+3 || max < 3 { return } - url, e := url.Parse(href) + commentURL, e := url.Parse(href) if e != nil { return } - url.Path, url.RawQuery, url.Fragment = "", "", "" - host := url.String() + commentURL.Path, commentURL.RawQuery, commentURL.Fragment = "", "", "" + host := commentURL.String() if host == "" { return } diff --git a/backend/app/store/formater_test.go b/backend/app/store/formater_test.go index cd33f24a..38465114 100644 --- a/backend/app/store/formater_test.go +++ b/backend/app/store/formater_test.go @@ -7,11 +7,11 @@ import ( "github.com/stretchr/testify/assert" ) -type mockConvertor struct{} +type mockConverter struct{} -func (m mockConvertor) Convert(text string) string { return text + "!converted" } +func (m mockConverter) Convert(text string) string { return text + "!converted" } -func TestFormater_FormatText(t *testing.T) { +func TestFormatter_FormatText(t *testing.T) { tbl := []struct { in, out string }{ @@ -22,18 +22,18 @@ func TestFormater_FormatText(t *testing.T) { "http://127.0.0.1/some-long-link/12345/678901234567890", "

http://127.0.0.1/some-long-link/12345/6789012...

\n!converted", }, } - f := NewCommentFormater(mockConvertor{}) + f := NewCommentFormatter(mockConverter{}) for n, tt := range tbl { assert.Equal(t, tt.out, f.FormatText(tt.in), "check #%d", n) } } -func TestFormater_FormatTextNoConvertor(t *testing.T) { - f := NewCommentFormater() +func TestFormatter_FormatTextNoConvertor(t *testing.T) { + f := NewCommentFormatter() assert.Equal(t, "

12345

\n", f.FormatText("12345")) } -func TestFormater_FormatComment(t *testing.T) { +func TestFormatter_FormatComment(t *testing.T) { comment := Comment{ Text: `blah`, User: User{ID: "username"}, @@ -47,14 +47,14 @@ func TestFormater_FormatComment(t *testing.T) { Votes: map[string]bool{"uu": true}, } - f := NewCommentFormater(mockConvertor{}) + f := NewCommentFormatter(mockConverter{}) exp := comment exp.Text = "

blah

\n!converted" assert.Equal(t, exp, f.Format(comment)) } -func TestFormater_ShortenAutoLinks(t *testing.T) { - f := NewCommentFormater(nil) +func TestFormatter_ShortenAutoLinks(t *testing.T) { + f := NewCommentFormatter(nil) tbl := []struct { max int in, out string From 92ef9882a8ced9028659c74fedf18b7ef3686ec1 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 11 Aug 2018 11:46:48 -0500 Subject: [PATCH 48/93] simplify readme a little bit --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dfaf6f04..6cb146bf 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * Social login via Google, Facebook, Github and Yandex * Multi-level nested comments with both tree and plain presentations -* Import from disqus, wordpress +* Import from disqus and wordpress * Markdown support * Moderator can remove comments and block users * Voting, pinning and verification system @@ -19,7 +19,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * Integration with automatic ssl via [nginx-le](https://github.com/umputun/nginx-le) * [Privacy focused](#privacy) ----- +# - [Install](#install) - [Backend](#backend) @@ -54,7 +54,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi - [Admin](#admin) - [Privacy](#privacy) - [Technical details](#technical-details) ----- + ## Install @@ -548,14 +548,14 @@ _all admin calls require auth and admin privilege_ ## Privacy * Remark42 is trying to be very sensitive to any private or semi-private information. -* Authentication requesting the lowest (minimal) possible scope from providers. All extra information returned by them dropped immediately and not stored in any form. -* Generally remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied. +* Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form. +* Generally, remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied. * There is no tracking of any sort. -* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing Cross-Site Request Forgery +* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing CSRF. * There is no cross-site login, i.e., user's behavior can't be analyzed across independent sites running remark42. * There are no third-party analytic services involved. * User can request all information remark42 knows about and export to gz file. -* Supported complete cleanup of all information related to user activity on demand. +* Supported complete cleanup of all information related to user's activity. * Cookie lifespan can be restricted to session-only. * All potentially sensitive data stored by remark42 hashed and encrypted. From 30ff1dacdee72f762b9b0596feb0304a1ab3d57e Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 12 Aug 2018 14:24:46 -0500 Subject: [PATCH 49/93] feature/artifact (#190) * embed assets for web with separate dockerfile * add artifact dockerfile * add artifacts upload to github release * fix statik params and add make for local builds * fix docker image name to match compose * fix typos --- .drone.yml | 11 + Dockerfile.artifacts | 87 ++++++++ Makefile | 11 + backend/Gopkg.lock | 9 + backend/app/rest/api/rest.go | 22 +- backend/app/rest/api/rest_private_test.go | 11 +- backend/app/store/comment.go | 2 +- backend/app/store/formater_test.go | 4 +- backend/app/store/user.go | 8 +- .../vendor/github.com/rakyll/statik/LICENSE | 202 ++++++++++++++++++ .../vendor/github.com/rakyll/statik/fs/fs.go | 146 +++++++++++++ 11 files changed, 497 insertions(+), 16 deletions(-) create mode 100644 Dockerfile.artifacts create mode 100644 Makefile create mode 100644 backend/vendor/github.com/rakyll/statik/LICENSE create mode 100644 backend/vendor/github.com/rakyll/statik/fs/fs.go diff --git a/.drone.yml b/.drone.yml index 1ba6f730..2e375596 100644 --- a/.drone.yml +++ b/.drone.yml @@ -42,6 +42,17 @@ pipeline: when: event: tag + artifacts_tag: + image: plugins/docker + dockerfile: Dockerfile.artifacts + build_args: + - DRONE=${DRONE} + - DRONE_TAG=${DRONE_TAG} + - DRONE_COMMIT=${DRONE_COMMIT} + - GITHUB_TOKEN=${GITHUB_TOKEN} + when: + event: tag + docker_branch: image: plugins/docker repo: umputun/remark42 diff --git a/Dockerfile.artifacts b/Dockerfile.artifacts new file mode 100644 index 00000000..2ab65127 --- /dev/null +++ b/Dockerfile.artifacts @@ -0,0 +1,87 @@ +FROM node:10.6-alpine as build-frontend-deps + +ARG CI +ENV SKIP_FRONTEND_TEST=true + +RUN apk add --no-cache --update git +ADD web/package.json /srv/web/package.json +ADD web/package-lock.json /srv/web/package-lock.json +RUN cd /srv/web && CI=true npm ci + +FROM node:10.6-alpine as build-frontend + +ARG CI +ARG NODE_ENV=production +ENV SKIP_FRONTEND_TEST=true + +COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules +ADD web /srv/web +RUN cd /srv/web && \ + npm run build && \ + rm -rf ./node_modules + + +FROM umputun/baseimage:buildgo-latest as build-backend + +ARG GITHUB_TOKEN +ENV SKIP_BACKEND_TEST=true + +WORKDIR /go/src/github.com/umputun/remark/backend +ADD backend /go/src/github.com/umputun/remark/backend +ADD README.md /go/src/github.com/umputun/remark/ +ADD LICENSE /go/src/github.com/umputun/remark/ + +COPY --from=build-frontend /srv/web web + +RUN \ + go get -v github.com/rakyll/statik && \ + statik --src=/go/src/github.com/umputun/remark/backend/web --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \ + ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go + +# if DRONE presented use DRONE_* git env to make version +RUN \ + if [ -z "$DRONE" ] ; then \ + echo "runs outside of drone" && version="local"; \ + else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \ + echo "version=$version" && \ + GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \ + GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \ + GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \ + GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \ + GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app + +RUN \ + if [ -z "$DRONE_TAG" ] ; then \ + echo "runs outside of drone" && tag=""; \ + else tag=_${DRONE_TAG}; fi && \ + apk add --no-cache --update zip && \ + tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \ + tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \ + tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \ + tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \ + zip remark${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md + +# upload to github +RUN \ + if [ -z "$DRONE_TAG" ] ; then \ + echo "skip upload to github" ; \ + else \ + curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \ + -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \ + "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \ + curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \ + -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \ + "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \ + curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \ + -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \ + "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \ + curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \ + -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \ + "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \ + curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \ + -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \ + "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi + +FROM alpine +COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/ +CMD ["sleep", "100"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..fc5b9690 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +OS=linux +ARCH=amd64 + +bin: + docker build -f Dockerfile.artifacts -t remark42.bin . + docker run -d --name=remark42.bin remark42.bin + docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42 + docker rm -f remark42.bin + +docker: + docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true . diff --git a/backend/Gopkg.lock b/backend/Gopkg.lock index c07a44c5..26598748 100644 --- a/backend/Gopkg.lock +++ b/backend/Gopkg.lock @@ -237,6 +237,14 @@ revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" +[[projects]] + digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e" + name = "github.com/rakyll/statik" + packages = ["fs"] + pruneopts = "UT" + revision = "19b88da8fc15428620782ba18f68423130e7ac7d" + version = "v0.1.3" + [[projects]] branch = "master" digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855" @@ -367,6 +375,7 @@ "github.com/nullrocks/identicon", "github.com/patrickmn/go-cache", "github.com/pkg/errors", + "github.com/rakyll/statik/fs", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/require", "golang.org/x/image/draw", diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 0a4b71cf..57c18d2d 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -20,6 +20,7 @@ import ( "github.com/go-chi/cors" "github.com/go-chi/render" "github.com/pkg/errors" + "github.com/rakyll/statik/fs" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" @@ -213,11 +214,24 @@ func (s *Rest) routes() chi.Router { return router } -// serves static files from /web +// serves static files from /web or embedded by statik func addFileServer(r chi.Router, path string, root http.FileSystem) { - log.Printf("[INFO] run file server for %s, path %s", root, path) + + var webFS http.Handler + + statikFS, err := fs.New() + if err == nil { + log.Printf("[INFO] run file server for %s, embedded", root) + webFS = http.FileServer(statikFS) + } + if err != nil { + log.Printf("[DEBUG] no embedded assets loaded, %s", err) + log.Printf("[INFO] run file server for %s, path %s", root, path) + webFS = http.FileServer(root) + } + origPath := path - fs := http.StripPrefix(path, http.FileServer(root)) + webFS = http.StripPrefix(path, webFS) if path != "/" && path[len(path)-1] != '/' { r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP) path += "/" @@ -231,7 +245,7 @@ func addFileServer(r chi.Router, path string, root http.FileSystem) { http.NotFound(w, r) return } - fs.ServeHTTP(w, r) + webFS.ServeHTTP(w, r) })) } diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index 26d09935..7b775ad8 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -24,10 +24,10 @@ func TestRest_Create(t *testing.T) { resp, err := post(t, ts.URL+"/api/v1/comment", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`) assert.Nil(t, err) - require.Equal(t, http.StatusCreated, resp.StatusCode) - b, err := ioutil.ReadAll(resp.Body) assert.Nil(t, err) + require.Equal(t, http.StatusCreated, resp.StatusCode, string(b)) + c := JSON{} err = json.Unmarshal(b, &c) assert.Nil(t, err) @@ -114,6 +114,7 @@ func TestRest_CreateRejected(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 401, resp.StatusCode) } + func TestRest_CreateAndGet(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) @@ -121,7 +122,7 @@ func TestRest_CreateAndGet(t *testing.T) { // create comment resp, err := post(t, ts.URL+"/api/v1/comment", - `{"text": "**test** *123* http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`) + `{"text": "**test** *123*\n\n http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`) require.Nil(t, err) require.Equal(t, http.StatusCreated, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) @@ -138,8 +139,8 @@ func TestRest_CreateAndGet(t *testing.T) { comment := store.Comment{} err = json.Unmarshal([]byte(res), &comment) assert.Nil(t, err) - assert.Equal(t, `

test 123 http://radio-t.com

`+"\n", comment.Text) - assert.Equal(t, "**test** *123* http://radio-t.com", comment.Orig) + assert.Equal(t, "

test 123

\n\n

http://radio-t.com

\n", comment.Text) + assert.Equal(t, "**test** *123*\n\n http://radio-t.com", comment.Orig) assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"}, comment.User) diff --git a/backend/app/store/comment.go b/backend/app/store/comment.go index 51c5fb38..a3b02557 100644 --- a/backend/app/store/comment.go +++ b/backend/app/store/comment.go @@ -94,7 +94,7 @@ func (c *Comment) SetDeleted(mode DeleteMode) { } } -// Sanitize clean dangerous html/js from the comment, shorten autolinks. +// Sanitize clean dangerous html/js from the comment func (c *Comment) Sanitize() { p := bluemonday.UGCPolicy() p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code") diff --git a/backend/app/store/formater_test.go b/backend/app/store/formater_test.go index 38465114..08685dc1 100644 --- a/backend/app/store/formater_test.go +++ b/backend/app/store/formater_test.go @@ -35,7 +35,7 @@ func TestFormatter_FormatTextNoConvertor(t *testing.T) { func TestFormatter_FormatComment(t *testing.T) { comment := Comment{ - Text: `blah`, + Text: "blah\n\nxyz", User: User{ID: "username"}, ParentID: "p123", ID: "123", @@ -49,7 +49,7 @@ func TestFormatter_FormatComment(t *testing.T) { f := NewCommentFormatter(mockConverter{}) exp := comment - exp.Text = "

blah

\n!converted" + exp.Text = "

blah

\n\n

xyz

\n!converted" assert.Equal(t, exp, f.Format(comment)) } diff --git a/backend/app/store/user.go b/backend/app/store/user.go index eec073e0..626521b4 100644 --- a/backend/app/store/user.go +++ b/backend/app/store/user.go @@ -37,7 +37,7 @@ func HashValue(val string, secret string) string { } key := []byte(secret) h := hmac.New(sha1.New, key) - return hashWithFailback(h, val) + return hashWithFallback(h, val) } // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases @@ -47,11 +47,11 @@ func EncodeID(id string) string { return id // already hashed or empty } h := sha1.New() - return hashWithFailback(h, id) + return hashWithFallback(h, id) } -// hashWithFailback tries to has val with hash.Hash and failback to crc if needed -func hashWithFailback(h hash.Hash, val string) string { +// hashWithFallback tries to has val with hash.Hash and failback to crc if needed +func hashWithFallback(h hash.Hash, val string) string { if _, err := io.WriteString(h, val); err != nil { // fail back to crc64 log.Printf("[WARN] can't hash id %s, %s", val, err) diff --git a/backend/vendor/github.com/rakyll/statik/LICENSE b/backend/vendor/github.com/rakyll/statik/LICENSE new file mode 100644 index 00000000..a4c5efd8 --- /dev/null +++ b/backend/vendor/github.com/rakyll/statik/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/backend/vendor/github.com/rakyll/statik/fs/fs.go b/backend/vendor/github.com/rakyll/statik/fs/fs.go new file mode 100644 index 00000000..82cda3a8 --- /dev/null +++ b/backend/vendor/github.com/rakyll/statik/fs/fs.go @@ -0,0 +1,146 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fs contains an HTTP file system that works with zip contents. +package fs + +import ( + "archive/zip" + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" +) + +var zipData string + +// file holds unzipped read-only file contents and file metadata. +type file struct { + os.FileInfo + data []byte +} + +type statikFS struct { + files map[string]file +} + +// Register registers zip contents data, later used to initialize +// the statik file system. +func Register(data string) { + zipData = data +} + +// New creates a new file system with the registered zip contents data. +// It unzips all files and stores them in an in-memory map. +func New() (http.FileSystem, error) { + if zipData == "" { + return nil, errors.New("statik/fs: no zip data registered") + } + zipReader, err := zip.NewReader(strings.NewReader(zipData), int64(len(zipData))) + if err != nil { + return nil, err + } + files := make(map[string]file) + for _, zipFile := range zipReader.File { + unzipped, err := unzip(zipFile) + if err != nil { + return nil, fmt.Errorf("statik/fs: error unzipping file %q: %s", zipFile.Name, err) + } + files["/"+zipFile.Name] = file{ + FileInfo: zipFile.FileInfo(), + data: unzipped, + } + } + return &statikFS{files: files}, nil +} + +func unzip(zf *zip.File) ([]byte, error) { + rc, err := zf.Open() + if err != nil { + return nil, err + } + defer rc.Close() + return ioutil.ReadAll(rc) +} + +// Open returns a file matching the given file name, or os.ErrNotExists if +// no file matching the given file name is found in the archive. +// If a directory is requested, Open returns the file named "index.html" +// in the requested directory, if that file exists. +func (fs *statikFS) Open(name string) (http.File, error) { + name = strings.Replace(name, "//", "/", -1) + f, ok := fs.files[name] + if ok { + return newHTTPFile(f, false), nil + } + // The file doesn't match, but maybe it's a directory, + // thus we should look for index.html + indexName := strings.Replace(name+"/index.html", "//", "/", -1) + f, ok = fs.files[indexName] + if !ok { + return nil, os.ErrNotExist + } + return newHTTPFile(f, true), nil +} + +func newHTTPFile(file file, isDir bool) *httpFile { + return &httpFile{ + file: file, + reader: bytes.NewReader(file.data), + isDir: isDir, + } +} + +// httpFile represents an HTTP file and acts as a bridge +// between file and http.File. +type httpFile struct { + file + + reader *bytes.Reader + isDir bool +} + +// Read reads bytes into p, returns the number of read bytes. +func (f *httpFile) Read(p []byte) (n int, err error) { + return f.reader.Read(p) +} + +// Seek seeks to the offset. +func (f *httpFile) Seek(offset int64, whence int) (ret int64, err error) { + return f.reader.Seek(offset, whence) +} + +// Stat stats the file. +func (f *httpFile) Stat() (os.FileInfo, error) { + return f, nil +} + +// IsDir returns true if the file location represents a directory. +func (f *httpFile) IsDir() bool { + return f.isDir +} + +// Readdir returns an empty slice of files, directory +// listing is disabled. +func (f *httpFile) Readdir(count int) ([]os.FileInfo, error) { + // directory listing is disabled. + return make([]os.FileInfo, 0), nil +} + +func (f *httpFile) Close() error { + return nil +} From 6cb4881039ba18ad701fc68bc6cc84ee0d97c1d5 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 12 Aug 2018 14:54:40 -0500 Subject: [PATCH 50/93] add info about local build procedure --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 6cb146bf..5ba62cc9 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * Export data to json with automatic backups * No external databases, everything embedded in a single data file * Fully dockerized and can be deployed in a single command +* Self-contained executable can be deployed directly to Linux, Windows and MacOS * Clean, lightweight and fully customizable UI * Multi-site mode from a single instance * Integration with automatic ssl via [nginx-le](https://github.com/umputun/nginx-le) @@ -42,6 +43,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi - [Comments](#comments) - [Last comments](#last-comments) - [Counter](#counter) + - [Build from the source](#build-from-the-source) - [Development](#development) - [Backend development](#backend-development) - [Frontend development](#frontend-development) @@ -337,6 +339,12 @@ and it will use `data-url` attribute to define the page with comments. Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined. +## Build from the source + +- to build docker container - `make docker`. This command will produce container `umputun/remark42`. +- to build a single binary for direct execution - `make OS= ARCH=`. This step will produce executable + `remark42` file with everything embedded. + ## Development You can use fully functional local version to develop and test both frontend & backend. From d5a75cfc00058c43d4b23f9dd04c4ac3d273c7e6 Mon Sep 17 00:00:00 2001 From: Anton Kosourov Date: Tue, 14 Aug 2018 07:33:00 +0500 Subject: [PATCH 51/93] Use comment formatter for wp importer (#192) * use comment formater in wp importer * remove debug code * change name to formatter --- backend/app/migrator/disqus.go | 4 +- backend/app/migrator/wordpress.go | 21 +++--- backend/app/migrator/wordpress_test.go | 90 +++++++++++++++++++++++++- 3 files changed, 103 insertions(+), 12 deletions(-) diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go index e9d657b3..ca6b2c5f 100644 --- a/backend/app/migrator/disqus.go +++ b/backend/app/migrator/disqus.go @@ -133,7 +133,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { Name: comment.AuthorName, IP: comment.IP, }, - Text: cleanText(comment.Message), + Text: d.cleanText(comment.Message), Timestamp: comment.CreatedAt, ParentID: comment.Pid.Val, } @@ -158,7 +158,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { return commentsCh } -func cleanText(text string) string { +func (*Disqus) cleanText(text string) string { text = strings.Replace(text, "\n", "", -1) text = strings.Replace(text, "\t", "", -1) return text diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go index 362ef881..5b2913b8 100644 --- a/backend/app/migrator/wordpress.go +++ b/backend/app/migrator/wordpress.go @@ -52,6 +52,13 @@ func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { return err } +// wpCommentConverter implements store.CommentConverter +type wpCommentConverter struct{} + +func (wpCommentConverter) Convert(text string) string { + return html.UnescapeString(text) // sanitize remains on comment create +} + // Import comments from WP and save to store func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) { @@ -92,6 +99,9 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { rejectedComments int // not approved }{} + commentConverter := new(wpCommentConverter) + commentFormatter := store.NewCommentFormatter(commentConverter) + go func() { for { t, err := decoder.Token() @@ -119,6 +129,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { if comment.PID == "0" { comment.PID = "" } + c := store.Comment{ ID: comment.ID, Locator: store.Locator{URL: item.Link, SiteID: siteID}, @@ -127,11 +138,11 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { Name: comment.Author, IP: comment.AuthorIP, }, - Text: w.cleanUnescapeText(comment.Content), // sanitize remains on comment create + Text: comment.Content, Timestamp: comment.Date.time, ParentID: comment.PID, } - commentsCh <- c + commentsCh <- commentFormatter.Format(c) stats.inpComments++ if stats.inpComments%1000 == 0 { log.Printf("[DEBUG] proccessed %d comments", stats.inpComments) @@ -146,9 +157,3 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { }() return commentsCh } - -func (w *WordPress) cleanUnescapeText(text string) string { - text = cleanText(text) - text = html.UnescapeString(text) - return text -} diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go index d1cef135..ae71e4d3 100644 --- a/backend/app/migrator/wordpress_test.go +++ b/backend/app/migrator/wordpress_test.go @@ -39,7 +39,7 @@ func TestWordPress_Import(t *testing.T) { assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP) ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14") assert.Equal(t, ts, c.Timestamp) - assert.Equal(t, c.Text, "Mekkatorque was over in that tent up to the right") + assert.Equal(t, c.Text, "

Mekkatorque was over in that tent up to the right

\n") posts, err := dataStore.List(siteID, 0, 0) assert.Nil(t, err) @@ -69,7 +69,7 @@ func TestWordPress_Convert(t *testing.T) { SiteID: "testWP", URL: "https://realmenweardress.es/2010/07/do-you-rp/", }, - Text: `[...] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. [...]`, + Text: `

[…] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. […]

` + "\n", User: store.User{ Name: "Wednesday Reading « Cynwise's Battlefield Manual", ID: "wordpress_" + store.EncodeID("Wednesday Reading « Cynwise's Battlefield Manual"), @@ -80,6 +80,25 @@ func TestWordPress_Convert(t *testing.T) { assert.Equal(t, exp1, comments[1]) } +func TestWP_Convert_MD(t *testing.T) { + wp := WordPress{} + ch := wp.convert(strings.NewReader(xmlTestWPmd), "siteID") + + comments := []store.Comment{} + for c := range ch { + comments = append(comments, c) + } + assert.Equal(t, 3, len(comments), "3 comments exported") + + assert.Equal(t, "

Row1
\nRow2

\n\n

Row4

\n", comments[0].Text) + + assert.Equal(t, "

markdown text

\n", comments[1].Text) + + expText := `

Row1 Link http://releases.rancher.com/os/latest markdown text blah

` + expText += "\n\n

Row3 markdownmd block

\n" + assert.Equal(t, expText, comments[2].Text) +} + var xmlTestWP = ` ` + +// parts of unused xml tags are omitted +var xmlTestWPmd = ` + + + + Deploying RancherOS on Vultr instances + https://realmenweardress.es/2016/07/deploying-rancheros-on-vultr-instances/ + + + 1 + + + https://eric.com + + + + + + + 0 + 0 + + + + 2 + + + https://eric.com + + + + + + + 0 + 0 + + + + 2 + + + https://eric.com + + + + + + + 0 + 0 + + + + + +` From 8ed75439ee6e2cc6789a2000945c9247e30b2a26 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 13 Aug 2018 21:42:37 -0500 Subject: [PATCH 52/93] fix typo in formatter name --- backend/app/store/{fomrater.go => formatter.go} | 0 backend/app/store/{formater_test.go => formatter_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename backend/app/store/{fomrater.go => formatter.go} (100%) rename backend/app/store/{formater_test.go => formatter_test.go} (100%) diff --git a/backend/app/store/fomrater.go b/backend/app/store/formatter.go similarity index 100% rename from backend/app/store/fomrater.go rename to backend/app/store/formatter.go diff --git a/backend/app/store/formater_test.go b/backend/app/store/formatter_test.go similarity index 100% rename from backend/app/store/formater_test.go rename to backend/app/store/formatter_test.go From 4e2895aa2dea449a4783e3d96af3bb6d70d3d751 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 13 Aug 2018 21:47:39 -0500 Subject: [PATCH 53/93] simplify wp converter, no need in special struct --- backend/app/migrator/wordpress.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go index 5b2913b8..4596d603 100644 --- a/backend/app/migrator/wordpress.go +++ b/backend/app/migrator/wordpress.go @@ -52,10 +52,8 @@ func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { return err } -// wpCommentConverter implements store.CommentConverter -type wpCommentConverter struct{} - -func (wpCommentConverter) Convert(text string) string { +// Convert satisfies formatter.CommentConverter +func (w *WordPress) Convert(text string) string { return html.UnescapeString(text) // sanitize remains on comment create } @@ -99,8 +97,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { rejectedComments int // not approved }{} - commentConverter := new(wpCommentConverter) - commentFormatter := store.NewCommentFormatter(commentConverter) + commentFormatter := store.NewCommentFormatter(w) go func() { for { From 8503e31f647145252a6a0874ad40be1dfc0b597c Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 15 Aug 2018 22:14:30 -0500 Subject: [PATCH 54/93] reformat imports --- backend/app/main.go | 4 ++-- backend/app/migrator/migrator.go | 1 + backend/app/migrator/remark.go | 1 + backend/app/rest/api/middleware.go | 1 + backend/app/rest/api/migrator.go | 2 +- backend/app/rest/auth/jwt.go | 2 +- backend/app/rest/cache/mongo.go | 3 +-- backend/app/rest/proxy/image.go | 1 + backend/app/store/engine/bolt_accessor.go | 3 +-- backend/app/store/service/service.go | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/backend/app/main.go b/backend/app/main.go index c15ba681..699a770b 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -16,16 +16,16 @@ import ( "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" "github.com/pkg/errors" - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/api" "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" + "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) diff --git a/backend/app/migrator/migrator.go b/backend/app/migrator/migrator.go index 422a25ab..ae80a52b 100644 --- a/backend/app/migrator/migrator.go +++ b/backend/app/migrator/migrator.go @@ -9,6 +9,7 @@ import ( "os" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store" ) diff --git a/backend/app/migrator/remark.go b/backend/app/migrator/remark.go index c15811dc..93b1505d 100644 --- a/backend/app/migrator/remark.go +++ b/backend/app/migrator/remark.go @@ -8,6 +8,7 @@ import ( "log" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store" ) diff --git a/backend/app/rest/api/middleware.go b/backend/app/rest/api/middleware.go index 2410dc6d..77185eb1 100644 --- a/backend/app/rest/api/middleware.go +++ b/backend/app/rest/api/middleware.go @@ -14,6 +14,7 @@ import ( "time" "github.com/go-chi/chi/middleware" + "github.com/umputun/remark/backend/app/rest" ) diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 3fdc3d9d..3010892a 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -15,11 +15,11 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/go-chi/render" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/cache" + "github.com/umputun/remark/backend/app/store/keys" ) // Migrator rest runs on unexposed port and available for local requests only diff --git a/backend/app/rest/auth/jwt.go b/backend/app/rest/auth/jwt.go index 0269ad0f..d6fd3c06 100644 --- a/backend/app/rest/auth/jwt.go +++ b/backend/app/rest/auth/jwt.go @@ -6,9 +6,9 @@ import ( "github.com/dgrijalva/jwt-go" "github.com/pkg/errors" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/keys" ) // JWT wraps jwt operations diff --git a/backend/app/rest/cache/mongo.go b/backend/app/rest/cache/mongo.go index 8f0c8e99..b0e23b5a 100644 --- a/backend/app/rest/cache/mongo.go +++ b/backend/app/rest/cache/mongo.go @@ -4,11 +4,10 @@ import ( "log" "time" - "github.com/go-pkgz/repeater" - "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "github.com/go-pkgz/mongo" + "github.com/go-pkgz/repeater" multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" ) diff --git a/backend/app/rest/proxy/image.go b/backend/app/rest/proxy/image.go index b24cbf04..284c8e13 100644 --- a/backend/app/rest/proxy/image.go +++ b/backend/app/rest/proxy/image.go @@ -11,6 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/go-chi/chi" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/rest" ) diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go index 9ee88c8e..1e14c79c 100644 --- a/backend/app/store/engine/bolt_accessor.go +++ b/backend/app/store/engine/bolt_accessor.go @@ -7,9 +7,8 @@ import ( "strings" "time" - "github.com/hashicorp/go-multierror" - "github.com/coreos/bbolt" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 98f05fcf..c9c35f03 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -6,10 +6,10 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" ) // DataStore wraps store.Interface with additional methods From a05b0b81c0dd46df35937998066d7bf01b8696fc Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 16 Aug 2018 18:37:57 -0500 Subject: [PATCH 55/93] add test for reported MD case https://github.com/umputun/remark/issues/156#issuecomment-413548318 --- backend/app/rest/api/migrator_test.go | 42 ++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 6de6342b..352ac3bb 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -2,6 +2,7 @@ package api import ( "compress/gzip" + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" @@ -13,6 +14,7 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" @@ -22,7 +24,7 @@ import ( ) func TestMigrator_Import(t *testing.T) { - srv, ts := prepImportSrv(t) + srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) @@ -42,11 +44,11 @@ func TestMigrator_Import(t *testing.T) { } func TestMigrator_ImportFromWP(t *testing.T) { - srv, ts := prepImportSrv(t) + srv, ds, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(xmlTestWP) + r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1)) client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress&secret=123456", r) @@ -59,10 +61,26 @@ func TestMigrator_ImportFromWP(t *testing.T) { b, err := ioutil.ReadAll(resp.Body) assert.Nil(t, err) assert.Equal(t, `{"size":3,"status":"ok"}`+"\n", string(b)) + + assert.NoError(t, ds.Interface.Close()) + + srvAccess, tsAccess := prep(t) + require.NotNil(t, srvAccess) + defer cleanup(ts, srvAccess) + + res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t") + require.Equal(t, 200, code) + comments := []store.Comment{} + err = json.Unmarshal([]byte(res), &comments) + assert.Nil(t, err) + assert.Equal(t, 3, len(comments), "should have 3 comments") + t.Logf("%+v", comments) + assert.Equal(t, "

Looks like http://releases.rancher.com/os/latest is no longer hosted - installs using this base-url are failing.

\n\n

I switched to Github with success:

\n\n
set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n
\n\n

Thanks for the article!

\n", + comments[0].Text) } func TestMigrator_ImportRejected(t *testing.T) { - srv, ts := prepImportSrv(t) + srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) @@ -78,7 +96,7 @@ func TestMigrator_ImportRejected(t *testing.T) { } func TestMigrator_Export(t *testing.T) { - srv, ts := prepImportSrv(t) + srv, _, ts := prepImportSrv(t) assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) @@ -125,7 +143,7 @@ func TestMigrator_Shutdown(t *testing.T) { assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms") } -func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) { +func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("123456")} @@ -140,7 +158,7 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) { routes := svc.routes() ts = httptest.NewServer(routes) - return svc, ts + return svc, dataStore, ts } func cleanupImportSrv(_ *Migrator, ts *httptest.Server) { @@ -292,7 +310,15 @@ var xmlTestWP = ` - + 13 From 8d9622cfd094afca3d2f7163558289d0a117a6a2 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 16 Aug 2018 18:48:34 -0500 Subject: [PATCH 56/93] shutdown after import test --- backend/app/rest/api/migrator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 352ac3bb..36b5d5eb 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -155,14 +155,14 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *http Cache: &cache.Nop{}, KeyStore: keys.NewStaticStore("123456"), } - routes := svc.routes() ts = httptest.NewServer(routes) return svc, dataStore, ts } -func cleanupImportSrv(_ *Migrator, ts *httptest.Server) { +func cleanupImportSrv(m *Migrator, ts *httptest.Server) { ts.Close() + m.Shutdown() os.Remove(testDb) } From 03eb57ef812d85f399d8fcdd6bce2f9aa9b0d5f9 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 16 Aug 2018 18:57:41 -0500 Subject: [PATCH 57/93] too many backticks by mistake --- backend/app/rest/api/migrator_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 36b5d5eb..5b62721a 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -14,12 +14,12 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/cache" + "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) @@ -310,7 +310,7 @@ var xmlTestWP = ` - Date: Fri, 17 Aug 2018 16:12:02 -0500 Subject: [PATCH 58/93] add a test for \n handling in wp, fix incorrect curl param caused removal of \n #156 --- backend/app/rest/api/middleware_test.go | 8 ++++++-- backend/scripts/import-wordpress.sh | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/app/rest/api/middleware_test.go b/backend/app/rest/api/middleware_test.go index aa10fb01..c2786057 100644 --- a/backend/app/rest/api/middleware_test.go +++ b/backend/app/rest/api/middleware_test.go @@ -39,13 +39,17 @@ func TestMiddleware_AppInfo(t *testing.T) { } func TestMiddleware_GetBodyAndUser(t *testing.T) { - req, err := http.NewRequest("GET", "http://example.com/request", strings.NewReader("body")) + req, err := http.NewRequest("GET", "http://example.com/request", strings.NewReader("body1\nbody2")) require.Nil(t, err) body, user := getBodyAndUser(req, []LoggerFlag{LogAll}) - assert.Equal(t, "body", body) + assert.Equal(t, "body1 body2", body) assert.Equal(t, "", user, "no user") + b, err := ioutil.ReadAll(req.Body) + assert.NoError(t, err) + assert.Equal(t, "body1\nbody2", string(b)) + req = rest.SetUserInfo(req, store.User{ID: "id1", Name: "user1"}) _, user = getBodyAndUser(req, []LoggerFlag{LogAll}) assert.Equal(t, ` - id1 "user1"`, user, "no user") diff --git a/backend/scripts/import-wordpress.sh b/backend/scripts/import-wordpress.sh index 1c224379..8e60dabf 100755 --- a/backend/scripts/import-wordpress.sh +++ b/backend/scripts/import-wordpress.sh @@ -1,5 +1,5 @@ #!/bin/sh set -e echo "import wordpress file $1 to site $2" -curl -X POST -H "Content-Type: application/xml" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=wordpress&secret=${SECRET}" +curl -X POST -H "Content-Type: application/xml" --data-binary @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=wordpress&secret=${SECRET}" echo "import completed" \ No newline at end of file From a6277d764a70c2d3abe81677abe9164f0f96e3cd Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 18 Aug 2018 23:09:17 -0500 Subject: [PATCH 59/93] add CommentConverterFunc to simplify injection of custom converters --- backend/app/store/formatter.go | 8 ++++++++ backend/app/store/formatter_test.go | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/backend/app/store/formatter.go b/backend/app/store/formatter.go index e51d7192..d1dec9c5 100644 --- a/backend/app/store/formatter.go +++ b/backend/app/store/formatter.go @@ -19,6 +19,14 @@ type CommentConverter interface { Convert(text string) string } +// CommentConverterFunc functional struct implementing CommentConverter +type CommentConverterFunc func(text string) string + +// Convert calls func for given text +func (f CommentConverterFunc) Convert(text string) string { + return f(text) +} + // NewCommentFormatter makes CommentFormatter func NewCommentFormatter(converters ...CommentConverter) *CommentFormatter { return &CommentFormatter{converters: converters} diff --git a/backend/app/store/formatter_test.go b/backend/app/store/formatter_test.go index 08685dc1..11f8b609 100644 --- a/backend/app/store/formatter_test.go +++ b/backend/app/store/formatter_test.go @@ -33,6 +33,12 @@ func TestFormatter_FormatTextNoConvertor(t *testing.T) { assert.Equal(t, "

12345

\n", f.FormatText("12345")) } +func TestFormatter_FormatTextConvertorFunc(t *testing.T) { + fn := CommentConverterFunc(func(text string) string { return "zz!" + text }) + f := NewCommentFormatter(fn) + assert.Equal(t, "zz!

12345

\n", f.FormatText("12345")) +} + func TestFormatter_FormatComment(t *testing.T) { comment := Comment{ Text: "blah\n\nxyz", From 419d2ae75e558d32ddb930e8beecc0e990d2ad28 Mon Sep 17 00:00:00 2001 From: Alexandr K Date: Thu, 23 Aug 2018 21:46:16 +0500 Subject: [PATCH 60/93] Add focus-visible polyfill, issue #187 (#193) * add focus-visible polyfill * remove unnecessary outline: none from styles * make buttons Pin, Block, Delete visible when focused * fix outline of user details close button * close user info by ESC * user info: don't focus the close button --- web/app/common/polyfills.js | 1 + .../comment/__action/comment__action.scss | 4 ---- .../comment/__control/comment__control.scss | 4 ---- .../comment/__controls/comment__controls.scss | 7 ++++--- .../comment/__vote/comment__vote.scss | 1 - .../input/__button/input__button.scss | 4 ---- .../root/__show-more/root__show-more.scss | 4 ---- web/app/embed.js | 18 +++++++++++++++--- web/iframe.html | 4 ++++ web/package-lock.json | 5 +++++ web/package.json | 1 + 11 files changed, 30 insertions(+), 23 deletions(-) diff --git a/web/app/common/polyfills.js b/web/app/common/polyfills.js index 05c15b1d..aad6a808 100644 --- a/web/app/common/polyfills.js +++ b/web/app/common/polyfills.js @@ -1,4 +1,5 @@ import 'core-js/es7/promise'; +import 'focus-visible'; export default function loadPolyfills() { const fillCoreJs = () => { diff --git a/web/app/components/comment/__action/comment__action.scss b/web/app/components/comment/__action/comment__action.scss index 9e3a0fda..39452f45 100644 --- a/web/app/components/comment/__action/comment__action.scss +++ b/web/app/components/comment/__action/comment__action.scss @@ -9,10 +9,6 @@ color: #31c7c5; } - &:focus { - outline: none; - } - + .comment__action { margin-left: 8px; } diff --git a/web/app/components/comment/__control/comment__control.scss b/web/app/components/comment/__control/comment__control.scss index dfc8b44c..53caeef3 100644 --- a/web/app/components/comment/__control/comment__control.scss +++ b/web/app/components/comment/__control/comment__control.scss @@ -10,8 +10,4 @@ &:hover { color: #0aa; } - - &:focus { - outline: none; - } } diff --git a/web/app/components/comment/__controls/comment__controls.scss b/web/app/components/comment/__controls/comment__controls.scss index 90b58b37..056a60ad 100644 --- a/web/app/components/comment/__controls/comment__controls.scss +++ b/web/app/components/comment/__controls/comment__controls.scss @@ -4,9 +4,10 @@ user-select: none; font-size: 14px; font-weight: 700; + opacity: 0; - @media (hover: hover) { - font-weight: 700; - opacity: 0; + &:hover, + &:focus-within { + opacity: 1; } } diff --git a/web/app/components/comment/__vote/comment__vote.scss b/web/app/components/comment/__vote/comment__vote.scss index dd3c9100..cf6dc84c 100644 --- a/web/app/components/comment/__vote/comment__vote.scss +++ b/web/app/components/comment/__vote/comment__vote.scss @@ -8,7 +8,6 @@ background: url('comment__vote.svg') center no-repeat; background-size: contain; cursor: pointer; - outline: none; &:hover { background-image: url('_selected/comment__vote_selected.svg'); diff --git a/web/app/components/input/__button/input__button.scss b/web/app/components/input/__button/input__button.scss index e8c4806c..92c6948b 100644 --- a/web/app/components/input/__button/input__button.scss +++ b/web/app/components/input/__button/input__button.scss @@ -7,10 +7,6 @@ border-radius: 2px; cursor: pointer; - &:focus { - outline: none; - } - &:disabled { opacity: 0.5; pointer-events: none; diff --git a/web/app/components/root/__show-more/root__show-more.scss b/web/app/components/root/__show-more/root__show-more.scss index c3186bf4..1b0fe800 100644 --- a/web/app/components/root/__show-more/root__show-more.scss +++ b/web/app/components/root/__show-more/root__show-more.scss @@ -18,8 +18,4 @@ &:focus { background: #0aa; } - - &:focus { - outline: none; - } } diff --git a/web/app/embed.js b/web/app/embed.js index efc07637..b801dcc0 100644 --- a/web/app/embed.js +++ b/web/app/embed.js @@ -60,6 +60,7 @@ function init() { node: null, back: null, closeEl: null, + iframe: null, style: null, init(user) { this.animationStop(); @@ -97,8 +98,6 @@ function init() { #${remarkRootId}-close { top: 0px; right: 400px; - width: 30px; - height: 30px; position: absolute; text-align: center; font-size: 25px; @@ -107,6 +106,7 @@ function init() { border-color: transparent; border-width: 0; padding: 0; + margin-right: 4px; background-color: transparent; } @media all and (max-width: 430px) { @@ -150,14 +150,19 @@ function init() { verticalscrolling="no" horizontalscrolling="no" />`; + this.iframe = this.node.querySelector('iframe'); + this.iframe.onload = () => { + this.iframe.contentDocument.addEventListener('keydown', this.onKeyDown); + }; this.node.appendChild(this.closeEl); document.body.appendChild(this.style); document.body.appendChild(this.back); document.body.appendChild(this.node); + document.addEventListener('keydown', this.onKeyDown); setTimeout(() => { this.back.setAttribute('data-animation', ''); this.node.setAttribute('data-animation', ''); - this.closeEl.focus(); + this.iframe.focus(); }, 400); }, close() { @@ -168,6 +173,7 @@ function init() { if (this.back) { this.back.removeAttribute('data-animation'); } + document.removeEventListener('keydown', this.onKeyDown); }, delay: null, events: ['', 'webkit', 'moz', 'MS', 'o'].map(prefix => (prefix ? `${prefix}TransitionEnd` : 'transitionend')), @@ -179,6 +185,12 @@ function init() { this.delay = setTimeout(this.animationStop, 1000); this.events.forEach(event => el.addEventListener(event, this.animationStop, false)); }, + onKeyDown(e) { + // ESCAPE key pressed + if (e.keyCode == 27) { + userInfo.close(); + } + }, animationStop() { const t = userInfo; if (!t.node) { diff --git a/web/iframe.html b/web/iframe.html index d65d6c24..e9dc515c 100644 --- a/web/iframe.html +++ b/web/iframe.html @@ -64,6 +64,10 @@ transform: scale(0); } } + + :focus:not(.focus-visible):not(.button) { + outline: none; + } diff --git a/web/package-lock.json b/web/package-lock.json index 285a58c3..cb6f9fa1 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -4800,6 +4800,11 @@ } } }, + "focus-visible": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/focus-visible/-/focus-visible-4.1.5.tgz", + "integrity": "sha512-yo/njtk/BB4Z2euzaZe3CZrg4u5s5uEi7ZwbHBJS2quHx51N0mmcx9nTIiImUGlgy+vf26d0CcQluahBBBL/Fw==" + }, "follow-redirects": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.4.1.tgz", diff --git a/web/package.json b/web/package.json index a49abbea..f04f5c52 100644 --- a/web/package.json +++ b/web/package.json @@ -62,6 +62,7 @@ "axios": "^0.18.0", "bem-react-helper": "^1.1.2", "core-js": "^2.5.7", + "focus-visible": "^4.1.5", "preact": "^8.2.9", "preact-redux": "^2.0.3", "redux": "^4.0.0" From ba6bab5b91688a0167d6a3696d2142c997236859 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 26 Aug 2018 21:45:04 -0500 Subject: [PATCH 61/93] Feature/cmd (#195) * support flags commands, move to cmd * fix target name * test for happy path importer * add export cmd * fix wrong import, lint warns * increase test timeout * add sellp to allow main test server to start * implement all cmds * handle backup/restore errors * fix import status check, hide secret from logs * backup cmd err tests * randimize test port * avoid dup code in Last controller * add target to make all bin archives * remove container in make * add smiple scripts to simplify commands, update readme * add docs on dockerless, enforce app user * add restore info * move last to lastCommentsScope const --- .dockerignore | 2 + .gitignore | 2 + Dockerfile | 17 +- Dockerfile.artifacts | 3 +- Makefile | 15 + README.md | 44 ++- backend/app/cmd/backup.go | 85 +++++ backend/app/cmd/backup_test.go | 71 ++++ backend/app/cmd/cmd.go | 66 ++++ backend/app/cmd/cmd_test.go | 39 ++ backend/app/cmd/import.go | 85 +++++ backend/app/cmd/import_test.go | 112 ++++++ backend/app/cmd/restore.go | 39 ++ backend/app/cmd/restore_test.go | 37 ++ backend/app/cmd/server.go | 446 ++++++++++++++++++++++ backend/app/cmd/server_test.go | 232 ++++++++++++ backend/app/cmd/testdata/import.txt | 3 + backend/app/cmd/testdata/import.txt.gz | Bin 0 -> 49 bytes backend/app/main.go | 450 +---------------------- backend/app/main_test.go | 227 ++---------- backend/app/rest/api/admin.go | 4 +- backend/app/rest/api/middleware.go | 22 ++ backend/app/rest/api/middleware_test.go | 16 + backend/app/rest/api/migrator.go | 2 +- backend/app/rest/api/rest.go | 4 +- backend/app/rest/api/rest_private.go | 4 +- backend/app/rest/api/rest_public.go | 12 +- backend/app/rest/api/rest_public_test.go | 1 + backend/app/rest/api/rest_test.go | 15 + backend/app/rest/api/rss.go | 4 +- backend/app/rest/auth/auth.go | 5 + backend/app/rest/proxy/avatar_test.go | 2 +- backend/scripts/backup.sh | 3 + backend/scripts/create-backup.sh | 9 - backend/scripts/import-disqus.sh | 5 - backend/scripts/import-wordpress.sh | 5 - backend/scripts/import.sh | 3 + backend/scripts/migrate-data.sh | 25 -- backend/scripts/restore-backup.sh | 14 - backend/scripts/restore.sh | 3 + compose-dev-backend.yml | 3 - compose-dev-frontend.yml | 3 - docker-compose.yml | 3 - start.sh => entrypoint.sh | 14 +- 44 files changed, 1400 insertions(+), 756 deletions(-) create mode 100644 backend/app/cmd/backup.go create mode 100644 backend/app/cmd/backup_test.go create mode 100644 backend/app/cmd/cmd.go create mode 100644 backend/app/cmd/cmd_test.go create mode 100644 backend/app/cmd/import.go create mode 100644 backend/app/cmd/import_test.go create mode 100644 backend/app/cmd/restore.go create mode 100644 backend/app/cmd/restore_test.go create mode 100644 backend/app/cmd/server.go create mode 100644 backend/app/cmd/server_test.go create mode 100644 backend/app/cmd/testdata/import.txt create mode 100644 backend/app/cmd/testdata/import.txt.gz create mode 100644 backend/scripts/backup.sh delete mode 100755 backend/scripts/create-backup.sh delete mode 100644 backend/scripts/import-disqus.sh delete mode 100755 backend/scripts/import-wordpress.sh create mode 100644 backend/scripts/import.sh delete mode 100755 backend/scripts/migrate-data.sh delete mode 100755 backend/scripts/restore-backup.sh create mode 100644 backend/scripts/restore.sh rename start.sh => entrypoint.sh (63%) diff --git a/.dockerignore b/.dockerignore index 3cd880c4..e6f1136f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,3 +19,5 @@ debug debug.test *.prof *.test +/bin/ +remark42 diff --git a/.gitignore b/.gitignore index e1385214..1ab85900 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ debug.test /rest-client.env.json .DS_Store .mongo +remark42 +/bin/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2f4a1a24..5006297e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ RUN \ echo "runs outside of drone" && version="local"; \ else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \ echo "version=$version" && \ - go build -o remark -ldflags "-X main.revision=${version} -s -w" ./app + go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app FROM node:10.6-alpine as build-frontend-deps @@ -88,16 +88,19 @@ FROM umputun/baseimage:app-latest WORKDIR /srv -ADD backend/scripts/*.sh /srv/ -ADD start.sh /srv/start.sh -RUN chmod +x /srv/*.sh +ADD entrypoint.sh /entrypoint.sh +ADD backend/scripts/backup.sh /usr/local/bin/backup +ADD backend/scripts/restore.sh /usr/local/bin/restore +ADD backend/scripts/import.sh /usr/local/bin/import +RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import -COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark /srv/ +COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42 COPY --from=build-frontend /srv/web/public/ /srv/web RUN chown -R app:app /srv +RUN ln -s /srv/remark42 /usr/bin/remark42 EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1 -CMD ["/srv/start.sh"] -ENTRYPOINT ["/init.sh"] +CMD ["server"] +ENTRYPOINT ["/entrypoint.sh"] diff --git a/Dockerfile.artifacts b/Dockerfile.artifacts index 2ab65127..1b739f87 100644 --- a/Dockerfile.artifacts +++ b/Dockerfile.artifacts @@ -59,7 +59,7 @@ RUN \ tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \ tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \ tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \ - zip remark${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md + zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md # upload to github RUN \ @@ -84,4 +84,5 @@ RUN \ FROM alpine COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/ +RUN ls -la /artifacts/* CMD ["sleep", "100"] diff --git a/Makefile b/Makefile index fc5b9690..27a983cf 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,24 @@ ARCH=amd64 bin: docker build -f Dockerfile.artifacts -t remark42.bin . + - @docker rm -f remark42.bin 2>/dev/null || exit 0 docker run -d --name=remark42.bin remark42.bin docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42 docker rm -f remark42.bin docker: docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true . + +deploy: + docker build -f Dockerfile.artifacts -t remark42.bin . + - @docker rm -f remark42.bin 2>/dev/null || exit 0 + - @mkdir -p bin + docker run -d --name=remark42.bin remark42.bin + docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz + docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz + docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz + docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz + docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip + docker rm -f remark42.bin + + diff --git a/README.md b/README.md index 5ba62cc9..b8756152 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi - [Install](#install) - [Backend](#backend) + - [With Docker](#with-docker) + - [Without docker](#without-docker) - [Parameters](#parameters) - [Required parameters](#required-parameters) - [Register oauth2 providers](#register-oauth2-providers) @@ -35,8 +37,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi - [Initial import from WordPress](#initial-import-from-wordpress) - [Backup and restore](#backup-and-restore) - [Automatic backups](#automatic-backups) - - [Schema migration](#schema-migration) - [Manual backup](#manual-backup) + - [Restore from backup](#restore-from-backup) - [Backup format](#backup-format) - [Admin users](#admin-users) - [Setup on your website](#setup-on-your-website) @@ -62,11 +64,21 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi ### Backend +#### With Docker + +_this is the recommended way to run remark42_ + * copy provided `docker-compose.yml` and customize for your needs -* prepare user id for container `` export USER=`id -u $USER` `` * make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments -* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d` -* alternatively compile from sources - `docker-compose build && docker-compose up -d` +* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d` +* alternatively compile from the sources - `docker-compose build && docker-compose up -d` + +#### Without docker + +* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads) +* unpack with `gunzip` (linux, mac os) or with `zip` (windows) +* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080` +* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]` #### Parameters @@ -139,7 +151,6 @@ services: - SECRET=abcd-123456-xyz-$%^& # secret key - AUTH_GITHUB_CID=12345667890 # oauth2 client ID - AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret - - USER=1001 # UID on the host machine, i.e `id -u` volumes: - ./var:/srv/var # persistent volume to store all remark42 data ``` @@ -200,13 +211,13 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co 1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details. 2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip .xml.gz`. -3. Run import command - `docker-compose exec remark42 /srv/import-disqus.sh .xml ` +3. Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}` #### Initial import from WordPress 1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. 2. Move this file to your remark42 host within `./var` -3. Run import command - `docker-compose exec remark42 /srv/import-wordpress.sh .xml ` +3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}` #### Backup and restore @@ -215,20 +226,19 @@ Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./ For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container: -`docker-compose exec remark42 /srv/restore-backup.sh {backup-filename.gz} {your site id}` - -##### Schema migration - -One special case for backup/restore is schema migration. Some versions or remark42 may extend or change the schema -and for such upgrades migration required. Provided migration script `scripts/migrate-data.sh` makes a fresh backup and then loads it back to your remark42 instance. - -`docker-compose exec remark42 /srv/migrate-data.sh {your site id}` +`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}` ##### Manual backup -In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` +In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default. -`docker-compose exec remark42 /srv/create-backup.sh {your site id}` +`docker exec -it remark42 backup -s {your site id}` + +##### Restore from backup + +Restore will clean all comments first and then will processed with complete import from a given file. + +`docker exec -it remark42 restore -f {backup file name} -s {your site id}` ##### Backup format diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go new file mode 100644 index 00000000..815e7794 --- /dev/null +++ b/backend/app/cmd/backup.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "time" + + "github.com/pkg/errors" +) + +// BackupCommand set of flags and command for export +// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. /) BACKUP_PATH ignored. +type BackupCommand struct { + ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` + ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"` + Site string `long:"site" env:"SITE" default:"remark" description:"site name"` + SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` +} + +// Execute runs export with ExportCommand parameters, entry point for "export" command +func (ec *BackupCommand) Execute(args []string) error { + log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site) + resetEnv("SECRET") + + fp := fileParser{site: ec.Site, path: ec.ExportPath, file: ec.ExportFile} + fname, err := fp.parse(time.Now()) + if err != nil { + return err + } + + log.Printf("[DEBUG] export file %s", fname) + + // prepare http client + client := http.Client{} + exportURL := fmt.Sprintf("%s/api/v1/admin/export?site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret) + req, err := http.NewRequest(http.MethodGet, exportURL, nil) + if err != nil { + return errors.Wrapf(err, "can't make export request for %s", exportURL) + } + ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout) + defer cancel() + + // get with timeout + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrapf(err, "request failed for %s", exportURL) + } + defer func() { + if err = resp.Body.Close(); err != nil { + log.Printf("[WARN] failed to close response, %s", err) + } + }() + + if resp.StatusCode >= 300 { + body, e := ioutil.ReadAll(resp.Body) + if e != nil { + body = []byte("") + } + return errors.Errorf("error response %q, %s", resp.Status, body) + } + + fh, err := os.Create(fname) + if err != nil { + return errors.Wrapf(err, "can't create backup file %s", fname) + } + defer func() { + if err = fh.Close(); err != nil { + log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err) + } + }() + + if _, err = io.Copy(fh, resp.Body); err != nil { + return errors.Wrapf(err, "failed to write backup file %s", fname) + } + + log.Printf("[INFO] export completed, file %s", fname) + return nil +} diff --git a/backend/app/cmd/backup_test.go b/backend/app/cmd/backup_test.go new file mode 100644 index 00000000..e3d56c11 --- /dev/null +++ b/backend/app/cmd/backup_test.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBackup_Execute(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/export") + assert.Equal(t, "GET", r.Method) + fmt.Fprint(w, "blah\nblah2\n12345678\n") + })) + defer ts.Close() + + cmd := BackupCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", + "--file={{.SITE}}-test.export", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) + defer os.Remove("/tmp/remark-test.export") + + data, err := ioutil.ReadFile("/tmp/remark-test.export") + require.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(data)) +} + +func TestBackup_ExecuteFailedStatus(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/export") + assert.Equal(t, "GET", r.Method) + w.WriteHeader(400) + fmt.Fprint(w, "some error") + })) + defer ts.Close() + + cmd := BackupCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", + "--file={{.SITE}}-test.export", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.EqualError(t, err, `error response "400 Bad Request", some error`) +} + +func TestBackup_ExecuteFailedWrite(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/export") + assert.Equal(t, "GET", r.Method) + fmt.Fprint(w, "blah\nblah2\n12345678\n") + })) + defer ts.Close() + + cmd := BackupCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", + "--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`) +} diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go new file mode 100644 index 00000000..15bee7c1 --- /dev/null +++ b/backend/app/cmd/cmd.go @@ -0,0 +1,66 @@ +// Package cmd has all top-level commands dispatched by main's flag.Parse +// The entry point of each command is Execute function +package cmd + +import ( + "bytes" + "log" + "os" + "path/filepath" + "strings" + "text/template" + "time" + + "github.com/pkg/errors" +) + +// fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format +type fileParser struct { + site string + file string + path string +} + +// parse apply template and also concat path and file. In case if file contains path separator path will be ignored +func (p *fileParser) parse(now time.Time) (string, error) { + + fileTemplate := struct { + YYYYMMDD string + YYYY string + YYYYMM string + MM string + DD string + TS string + UNIX int64 + SITE string + }{ + YYYYMMDD: now.Format("20060102"), + YYYY: now.Format("2006"), + YYYYMM: now.Format("200601"), + MM: now.Format("01"), + DD: now.Format("02"), + UNIX: now.Unix(), + SITE: p.site, + TS: now.Format("20060102T150405"), + } + + bb := bytes.Buffer{} + fname := p.file + if !strings.Contains(p.file, string(filepath.Separator)) { + fname = filepath.Join(p.path, p.file) + } + + if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil { + return "", errors.Wrapf(err, "failed to parse %q", fname) + } + return bb.String(), nil +} + +// resetEnv clears sensitive env vars +func resetEnv(envs ...string) { + for _, env := range envs { + if err := os.Unsetenv(env); err != nil { + log.Printf("[WARN] can't unset env %s, %s", env, err) + } + } +} diff --git a/backend/app/cmd/cmd_test.go b/backend/app/cmd/cmd_test.go new file mode 100644 index 00000000..de20b50d --- /dev/null +++ b/backend/app/cmd/cmd_test.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestExport_ParseFileName(t *testing.T) { + tbl := []struct { + p fileParser + res string + err bool + }{ + {fileParser{}, "", false}, + {fileParser{path: "/tmp/blah", file: "fname.gz"}, "/tmp/blah/fname.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYYMMDD}}.gz"}, + "/tmp/blah/fname-remark-20180821.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"}, + "/tmp/blah/fname-remark-2018-08.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"}, + "/tmp/fname-remark-2018-08.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.TS}}.gz"}, + "/tmp/fname-remark-20180821T212615.gz", false}, + {fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.XXX}}-{{.YYYY}}-{{.MM}}.gz"}, + "", true}, + } + + now := time.Date(2018, 8, 21, 21, 26, 15, 0, time.UTC) + for i, tt := range tbl { + r, err := tt.p.parse(now) + if tt.err { + assert.NotNil(t, err) + continue + } + assert.Equal(t, tt.res, r, "check #%d", i) + } +} diff --git a/backend/app/cmd/import.go b/backend/app/cmd/import.go new file mode 100644 index 00000000..13b4ef8b --- /dev/null +++ b/backend/app/cmd/import.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strings" + "time" + + "github.com/pkg/errors" +) + +// ImportCommand set of flags and command for import +type ImportCommand struct { + InputFile string `short:"f" long:"file" description:"input file name" required:"true"` + Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` + Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` + SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` + + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` +} + +// Execute runs import with ImportCommand parameters, entry point for "import" command +func (ic *ImportCommand) Execute(args []string) error { + log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site) + resetEnv("SECRET") + + reader, err := ic.reader(ic.InputFile) + if err != nil { + return errors.Wrapf(err, "can't open import file %s", ic.InputFile) + } + + client := http.Client{} + importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s", ic.URL, ic.Site, ic.Provider, ic.SharedSecret) + req, err := http.NewRequest(http.MethodPost, importURL, reader) + if err != nil { + return errors.Wrapf(err, "can't make import request for %s", importURL) + } + + ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout) + defer cancel() + + resp, err := client.Do(req.WithContext(ctx)) // closes reader + if err != nil { + return errors.Wrapf(err, "request failed for %s", importURL) + } + defer func() { + if err = resp.Body.Close(); err != nil { + log.Printf("[WARN] failed to close response, %s", err) + } + }() + if resp.StatusCode >= 300 { + return errors.Errorf("error response %s (%d)", resp.Status, resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "can't get response from importer") + } + + log.Printf("[INFO] import completed, status=%d, %s", resp.StatusCode, string(body)) + return nil +} + +// reader returns reader for file. For .gz file wraps with gunzip +func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) { + inpFile, err := os.Open(inp) + if err != nil { + return nil, errors.Wrapf(err, "import failed, can't open %s", inp) + } + + reader = inpFile + if strings.HasSuffix(ic.InputFile, ".gz") { + if reader, err = gzip.NewReader(inpFile); err != nil { + return nil, errors.Wrap(err, "can't make gz reader") + } + } + return reader, nil +} diff --git a/backend/app/cmd/import_test.go b/backend/app/cmd/import_test.go new file mode 100644 index 00000000..30188da5 --- /dev/null +++ b/backend/app/cmd/import_test.go @@ -0,0 +1,112 @@ +package cmd + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImport_Execute(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + body, err := ioutil.ReadAll(r.Body) + assert.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(body)) + + fmt.Fprintln(w, "some response") + fmt.Fprintln(w, string(body)) + })) + defer ts.Close() + + cmd := ImportCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) + + cmd = ImportCommand{} + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt.gz", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) +} + +func TestImport_ExecuteFailed(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + fmt.Fprintln(w, "some response") + })) + defer ts.Close() + + cmd := ImportCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import-no.txt", "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + t.Log(err) + assert.NotNil(t, err, "fail on no such file") + assert.True(t, strings.Contains(err.Error(), "no such file or directory")) + + cmd = ImportCommand{} + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", + "--url=http://127.0.0.1:12345"}) + require.Nil(t, err) + err = cmd.Execute(nil) + t.Log(err) + assert.NotNil(t, err, "fail on connection refused") + assert.True(t, strings.Contains(err.Error(), "connection refused")) + + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%+v", r) + w.WriteHeader(400) + fmt.Fprintln(w, "some response with 400") + })) + defer ts2.Close() + cmd = ImportCommand{} + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts2.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + t.Log(err) + assert.NotNil(t, err) +} + +func TestImport_ExecuteTimeout(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + body, err := ioutil.ReadAll(r.Body) + assert.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(body)) + time.Sleep(500 * time.Millisecond) + fmt.Fprintln(w, "some response") + fmt.Fprintln(w, string(body)) + + })) + defer ts.Close() + + cmd := ImportCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", + "--url=" + ts.URL, "--timeout=300ms"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "deadline exceeded")) +} diff --git a/backend/app/cmd/restore.go b/backend/app/cmd/restore.go new file mode 100644 index 00000000..c8e42461 --- /dev/null +++ b/backend/app/cmd/restore.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "log" + "time" +) + +// RestoreCommand set of flags and command for restore from backup +type RestoreCommand struct { + ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` + ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"` + + Site string `long:"site" env:"SITE" default:"remark" description:"site name"` + SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` +} + +// Execute runs import with RestoreCommand parameters, entry point for "restore" command +// uses ImportCommand with constructed full file name +func (rc *RestoreCommand) Execute(args []string) error { + log.Printf("[INFO] restore %s, site %s", rc.ImportFile, rc.Site) + resetEnv("SECRET") + + fp := fileParser{site: rc.Site, path: rc.ImportPath, file: rc.ImportFile} + fname, err := fp.parse(time.Now()) + if err != nil { + return err + } + importer := ImportCommand{ + InputFile: fname, + Site: rc.Site, + Provider: "native", + SharedSecret: rc.SharedSecret, + Timeout: rc.Timeout, + URL: rc.URL, + } + return importer.Execute(args) +} diff --git a/backend/app/cmd/restore_test.go b/backend/app/cmd/restore_test.go new file mode 100644 index 00000000..820968fb --- /dev/null +++ b/backend/app/cmd/restore_test.go @@ -0,0 +1,37 @@ +package cmd + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRestore_Execute(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.URL.Path, "/api/v1/admin/import") + assert.Equal(t, "POST", r.Method) + assert.Equal(t, "native", r.URL.Query().Get("provider")) + body, err := ioutil.ReadAll(r.Body) + assert.Nil(t, err) + assert.Equal(t, "blah\nblah2\n12345678\n", string(body)) + + fmt.Fprintln(w, "some response") + fmt.Fprintln(w, string(body)) + })) + defer ts.Close() + + cmd := RestoreCommand{} + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=testdata", "--file=import.txt", + "--url=" + ts.URL}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) +} diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go new file mode 100644 index 00000000..c85c3889 --- /dev/null +++ b/backend/app/cmd/server.go @@ -0,0 +1,446 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/coreos/bbolt" + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/migrator" + "github.com/umputun/remark/backend/app/rest/api" + "github.com/umputun/remark/backend/app/rest/auth" + "github.com/umputun/remark/backend/app/rest/cache" + "github.com/umputun/remark/backend/app/rest/proxy" + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/avatar" + "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" + "github.com/umputun/remark/backend/app/store/service" +) + +// ServerOpts with command line flags and env +type ServerOpts struct { + RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` + SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` + + Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` + Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` + Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` + Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` + Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` + + Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` + Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` + AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"` + DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"` + BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"` + MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` + ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"` + MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"` + LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"` + CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"` + ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"` + EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"` + Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` + WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` + // Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` + + Auth struct { + TTL struct { + JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"` + Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"` + } `group:"ttl" namespace:"ttl" env-namespace:"TTL"` + Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"` + Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"` + Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"` + Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"` + Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"` + } `group:"auth" namespace:"auth" env-namespace:"AUTH"` +} + +// AuthGroup defines options group for auth params +type AuthGroup struct { + CID string `long:"cid" env:"CID" description:"OAuth client ID"` + CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"` +} + +// StoreGroup defines options group for store params +type StoreGroup struct { + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"` + Bolt struct { + Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` + Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` + } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` +} + +// AvatarGroup defines options group for avatar params +type AvatarGroup struct { + Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"` + FS struct { + Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` + } `group:"fs" namespace:"fs" env-namespace:"FS"` + RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` +} + +// CacheGroup defines options group for cache params +type CacheGroup struct { + Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"` + Max struct { + Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` + Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` + Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"` + } `group:"max" namespace:"max" env-namespace:"MAX"` +} + +// MongoGroup holds all mongo params, used by store, avatar and cache +type MongoGroup struct { + URL string `long:"url" env:"URL" description:"mongo url"` + DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` +} + +// KeyGroup defines options group for key params +type KeyGroup struct { + Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` +} + +// Revision sets from main +var Revision = "unknown" + +// serverApp holds all active objects +type serverApp struct { + *ServerOpts + restSrv *api.Rest + migratorSrv *api.Migrator + exporter migrator.Exporter + devAuth *auth.DevAuthServer + dataService *service.DataStore + terminated chan struct{} +} + +// Execute is the entry point for "server" command, called by flag parser +func (s *ServerOpts) Execute(args []string) error { + log.Print("[INFO] start remark42 server") + resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC") + + ctx, cancel := context.WithCancel(context.Background()) + go func() { // catch signal and invoke graceful termination + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop + log.Print("[WARN] interrupt signal") + cancel() + }() + + app, err := newServerApp(s) + if err != nil { + log.Fatalf("[ERROR] failed to setup application, %+v", err) + } + if err = app.run(ctx); err != nil { + log.Printf("[INFO] remark terminated with error %+v", err) + return err + } + log.Printf("[INFO] remark terminated") + return nil +} + +// newServerApp prepares application and return it with all active parts +// doesn't start anything +func newServerApp(opts *ServerOpts) (*serverApp, error) { + + if err := makeDirs(opts.BackupLocation); err != nil { + return nil, err + } + + if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") { + return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL) + } + + storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites) + if err != nil { + return nil, err + } + + keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret) + if err != nil { + return nil, err + } + + dataService := &service.DataStore{ + Interface: storeEngine, + EditDuration: opts.EditDuration, + KeyStore: keyStore, + MaxCommentSize: opts.MaxCommentSize, + Admins: opts.Admins, + } + + loadingCache, err := makeCache(opts.Cache, opts.Mongo) + if err != nil { + return nil, err + } + + // token TTL is 5 minutes, inactivity interval 7+ days by default + jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) + + avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) + if err != nil { + return nil, errors.Wrap(err, "failed to make avatar store") + } + avatarProxy := &proxy.Avatar{ + Store: avatarStore, + RoutePath: "/api/v1/avatar", + RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), + } + + exporter := &migrator.Remark{DataStore: dataService} + + migr := &api.Migrator{ + Version: Revision, + Cache: loadingCache, + NativeImporter: &migrator.Remark{DataStore: dataService}, + DisqusImporter: &migrator.Disqus{DataStore: dataService}, + WordPressImporter: &migrator.WordPress{DataStore: dataService}, + NativeExported: &migrator.Remark{DataStore: dataService}, + KeyStore: keyStore, + } + + authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) + imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} + commentFormatter := store.NewCommentFormatter(imgProxy) + + srv := &api.Rest{ + Version: Revision, + DataService: dataService, + Exporter: exporter, + WebRoot: opts.WebRoot, + RemarkURL: opts.RemarkURL, + ImageProxy: imgProxy, + CommentFormatter: commentFormatter, + AvatarProxy: avatarProxy, + ReadOnlyAge: opts.ReadOnlyAge, + SharedSecret: opts.SharedSecret, + Authenticator: auth.Authenticator{ + JWTService: jwtService, + AdminEmail: opts.AdminEmail, + Providers: authProviders, + DevPasswd: opts.DevPasswd, + PermissionChecker: dataService, + }, + Cache: loadingCache, + } + + // no admin email, use admin@domain + if srv.Authenticator.AdminEmail == "" { + if u, err := url.Parse(opts.RemarkURL); err == nil { + srv.Authenticator.AdminEmail = "admin@" + u.Host + } + } + + srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore + + var devAuth *auth.DevAuthServer + if opts.Auth.Dev { + devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]} + } + + tch := make(chan struct{}) + return &serverApp{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService, + ServerOpts: opts, terminated: tch}, nil +} + +// Run all application objects +func (a *serverApp) run(ctx context.Context) error { + if a.DevPasswd != "" { + log.Printf("[WARN] running in dev mode") + } + + go func() { + // shutdown on context cancellation + <-ctx.Done() + a.restSrv.Shutdown() + a.migratorSrv.Shutdown() + if a.devAuth != nil { + a.devAuth.Shutdown() + } + if e := a.dataService.Close(); e != nil { + log.Printf("[WARN] failed to close store, %s", e) + } + + }() + a.activateBackup(ctx) // runs in goroutine for each site + go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only + if a.Auth.Dev { + go a.devAuth.Run() // dev oauth2 server on :8084 + } + a.restSrv.Run(a.Port) + close(a.terminated) + return nil +} + +// Wait for application completion (termination) +func (a *serverApp) Wait() { + <-a.terminated +} + +// activateBackup runs background backups for each site +func (a *serverApp) activateBackup(ctx context.Context) { + for _, siteID := range a.Sites { + backup := migrator.AutoBackup{ + Exporter: a.exporter, + BackupLocation: a.BackupLocation, + SiteID: siteID, + KeepMax: a.MaxBackupFiles, + Duration: 24 * time.Hour, + } + go backup.Do(ctx) + } +} + +// makeDataStore creates store for all sites +func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) { + switch group.Type { + case "bolt": + if err = makeDirs(group.Bolt.Path); err != nil { + return nil, errors.Wrap(err, "failed to create bolt store") + } + sites := []engine.BoltSite{} + for _, site := range siteNames { + sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)}) + } + result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...) + case "mongo": + mgServer, e := makeMongo(mg) + if e != nil { + return result, errors.Wrap(e, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, mg.DB, "") + result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) + default: + return nil, errors.Errorf("unsupported store type %s", group.Type) + } + return result, errors.Wrap(err, "can't initialize data store") +} + +func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { + switch group.Type { + case "fs": + if err := makeDirs(group.FS.Path); err != nil { + return nil, err + } + return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil + case "mongo": + mgServer, err := makeMongo(mg) + if err != nil { + return nil, errors.Wrap(err, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, mg.DB, "") + return avatar.NewGridFS(conn, group.RszLmt), nil + } + return nil, errors.Errorf("unsupported avatar store type %s", group.Type) +} + +func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) { + switch group.Type { + case "shared": + return keys.NewStaticStore(sharedSecret), nil + default: + return nil, errors.Errorf("unsupported key store type %s", group.Type) + } +} + +func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { + switch group.Type { + case "mem": + return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), + cache.MaxKeys(group.Max.Items)) + case "mongo": + mgServer, err := makeMongo(mg) + if err != nil { + return nil, errors.Wrap(err, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, mg.DB, "cache") + return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), + cache.MaxKeys(group.Max.Items)) + } + return nil, errors.Errorf("unsupported cache type %s", group.Type) +} + +// mkdir -p for all dirs +func makeDirs(dirs ...string) error { + + // exists returns whether the given file or directory exists or not + exists := func(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return true, err + } + + for _, dir := range dirs { + ex, err := exists(dir) + if err != nil { + return errors.Wrapf(err, "can't check directory status for %s", dir) + } + if !ex { + if e := os.MkdirAll(dir, 0700); e != nil { + return errors.Wrapf(err, "can't make directory %s", dir) + } + } + } + return nil +} + +func makeMongo(mg MongoGroup) (result *mongo.Server, err error) { + if mg.URL == "" { + return nil, errors.New("no mongo URL provided") + } + return mongo.NewServerWithURL(mg.URL, 10*time.Second) +} + +func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts *ServerOpts) []auth.Provider { + + makeParams := func(cid, secret string) auth.Params { + return auth.Params{ + JwtService: jwtService, + AvatarProxy: avatarProxy, + RemarkURL: opts.RemarkURL, + Cid: cid, + Csecret: secret, + PermissionChecker: ds, + } + } + + providers := []auth.Provider{} + if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" { + providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC))) + } + if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" { + providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC))) + } + if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" { + providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC))) + } + if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" { + providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC))) + } + if opts.Auth.Dev { + providers = append(providers, auth.NewDev(makeParams("", ""))) + } + + if len(providers) == 0 { + log.Printf("[WARN] no auth providers defined") + } + return providers +} diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go new file mode 100644 index 00000000..5f25501e --- /dev/null +++ b/backend/app/cmd/server_test.go @@ -0,0 +1,232 @@ +package cmd + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strings" + "syscall" + "testing" + "time" + + "github.com/globalsign/mgo" + "github.com/go-pkgz/mongo" + "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServerApp(t *testing.T) { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + o.Port = 18080 + return o + }) + + go func() { _ = app.run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:18080/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + // add comment + resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json", + strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`)) + require.Nil(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + body, _ = ioutil.ReadAll(resp.Body) + t.Log(string(body)) + + assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email") + + app.Wait() +} + +func TestServerApp_DevMode(t *testing.T) { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + o.Port = 18085 + o.DevPasswd = "password" + o.Auth.Dev = true + return o + }) + + go func() { _ = app.run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider") + assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider") + // send ping + resp, err := http.Get("http://localhost:18085/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + app.Wait() +} + +func TestServerApp_WithMongo(t *testing.T) { + + mongoURL := os.Getenv("MONGO_TEST") + if mongoURL == "" { + mongoURL = "mongodb://localhost:27017/test" + } + if mongoURL == "skip" { + t.Skip("skip mongo app test") + } + + opts := ServerOpts{} + // prepare options + p := flags.NewParser(&opts, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com", + "--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) + require.Nil(t, err) + opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" + opts.BackupLocation = "/tmp" + + // create app + app, err := newServerApp(&opts) + require.Nil(t, err) + + defer func() { + s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second) + assert.NoError(t, err) + conn := mongo.NewConnection(s, "test_remark", "") + _ = conn.WithDB(func(dbase *mgo.Database) error { + assert.NoError(t, dbase.DropDatabase()) + return nil + }) + }() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Second) + log.Print("[TEST] terminate app") + cancel() + }() + go func() { _ = app.run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:12345/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + app.Wait() +} + +func TestServerApp_Failed(t *testing.T) { + opts := ServerOpts{} + p := flags.NewParser(&opts, flags.Default) + + // RO bolt location + _, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", + "--store.bolt.path=/dev/null"}) + assert.Nil(t, err) + _, err = newServerApp(&opts) + assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ + "open /dev/null/remark.db: not a directory") + t.Log(err) + + // RO backup location + opts = ServerOpts{} + _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp", + "--backup=/dev/null/not-writable"}) + assert.Nil(t, err) + _, err = newServerApp(&opts) + assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory") + t.Log(err) + + // invalid url + opts = ServerOpts{} + _, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"}) + assert.Nil(t, err) + _, err = newServerApp(&opts) + assert.EqualError(t, err, "invalid remark42 url demo.remark42.com") + t.Log(err) + + opts = ServerOpts{} + _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"}) + assert.NotNil(t, err, "blah is invalid type") + + opts.Store.Type = "blah" + _, err = newServerApp(&opts) + assert.EqualError(t, err, "unsupported store type blah") + t.Log(err) +} + +func TestServerApp_Shutdown(t *testing.T) { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + o.Port = 18090 + return o + }) + st := time.Now() + err := app.run(ctx) + assert.Nil(t, err) + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") + app.Wait() +} + +func TestServerApp_MainSignal(t *testing.T) { + + go func() { + time.Sleep(100 * time.Millisecond) + err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM) + require.Nil(t, err) + }() + st := time.Now() + + s := ServerOpts{} + p := flags.NewParser(&s, flags.Default) + args := []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", + "--port=18100", "--url=https://demo.remark42.com"} + _, err := p.ParseArgs(args) + require.Nil(t, err) + s.Execute(args) + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") +} + +func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerOpts) ServerOpts) (*serverApp, context.Context) { + opts := ServerOpts{} + // prepare options + p := flags.NewParser(&opts, flags.Default) + _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"}) + require.Nil(t, err) + opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp" + opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port) + opts.Store.Bolt.Timeout = 10 * time.Second + opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" + opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid" + opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid" + opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid" + opts.BackupLocation = "/tmp" + opts = fn(opts) + + os.Remove(opts.Store.Bolt.Path + "/remark.db") + + // create app + app, err := newServerApp(&opts) + require.Nil(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(duration) + log.Print("[TEST] terminate app") + cancel() + }() + return app, ctx +} diff --git a/backend/app/cmd/testdata/import.txt b/backend/app/cmd/testdata/import.txt new file mode 100644 index 00000000..f9920896 --- /dev/null +++ b/backend/app/cmd/testdata/import.txt @@ -0,0 +1,3 @@ +blah +blah2 +12345678 diff --git a/backend/app/cmd/testdata/import.txt.gz b/backend/app/cmd/testdata/import.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..8691c9afcc4664add0b040afccf1bb151c1ab3d7 GIT binary patch literal 49 zcmb2|=HNJ$RTIs?oS9pYUsR%3Qc=R-eaiF911}~ax0p7!V`|INo{7aU%$}z1Bf`MI F006Z35YPYs literal 0 HcmV?d00001 diff --git a/backend/app/main.go b/backend/app/main.go index 699a770b..df619011 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -1,460 +1,48 @@ package main import ( - "context" "fmt" "log" - "net/url" "os" - "os/signal" - "strings" - "syscall" - "time" - "github.com/coreos/bbolt" - "github.com/go-pkgz/mongo" "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" - "github.com/pkg/errors" - "github.com/umputun/remark/backend/app/migrator" - "github.com/umputun/remark/backend/app/rest/api" - "github.com/umputun/remark/backend/app/rest/auth" - "github.com/umputun/remark/backend/app/rest/cache" - "github.com/umputun/remark/backend/app/rest/proxy" - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/avatar" - "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" - "github.com/umputun/remark/backend/app/store/service" + "github.com/umputun/remark/backend/app/cmd" ) -// Opts with command line flags and env +// Opts has all commands type Opts struct { - RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` - SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` + ServerCmd cmd.ServerOpts `command:"server"` + ImportCmd cmd.ImportCommand `command:"import"` + BackupCmd cmd.BackupCommand `command:"backup"` + RestoreCmd cmd.RestoreCommand `command:"restore"` - Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` - Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` - Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` - Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` - Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` - - Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` - Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` - AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"` - DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"` - BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"` - MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` - ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"` - MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"` - LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"` - CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"` - ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"` - EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"` - Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` - WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` - Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` - - Auth struct { - TTL struct { - JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"` - Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"` - } `group:"ttl" namespace:"ttl" env-namespace:"TTL"` - Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"` - Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"` - Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"` - Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"` - Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"` - } `group:"auth" namespace:"auth" env-namespace:"AUTH"` -} - -// AuthGroup defines options group for auth params -type AuthGroup struct { - CID string `long:"cid" env:"CID" description:"OAuth client ID"` - CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"` -} - -// StoreGroup defines options group for store params -type StoreGroup struct { - Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"` - Bolt struct { - Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` - Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` - } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` -} - -// AvatarGroup defines options group for avatar params -type AvatarGroup struct { - Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"` - FS struct { - Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` - } `group:"fs" namespace:"fs" env-namespace:"FS"` - RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` -} - -// CacheGroup defines options group for cache params -type CacheGroup struct { - Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"` - Max struct { - Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` - Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` - Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"` - } `group:"max" namespace:"max" env-namespace:"MAX"` -} - -// MongoGroup holds all mongo params, used by store, avatar and cache -type MongoGroup struct { - URL string `long:"url" env:"URL" description:"mongo url"` - DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` -} - -// KeyGroup defines options group for key params -type KeyGroup struct { - Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` + Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` } var revision = "unknown" -// Application holds all active objects -type Application struct { - Opts - restSrv *api.Rest - migratorSrv *api.Migrator - exporter migrator.Exporter - devAuth *auth.DevAuthServer - dataService *service.DataStore - terminated chan struct{} -} - func main() { - fmt.Printf("remark %s\n", revision) + fmt.Printf("remark42 %s\n", revision) + cmd.Revision = revision var opts Opts p := flags.NewParser(&opts, flags.Default) - if _, e := p.ParseArgs(os.Args[1:]); e != nil { - os.Exit(1) - } - - setupLog(opts.Dbg) - log.Print("[INFO] started remark") - resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC") - - ctx, cancel := context.WithCancel(context.Background()) - go func() { // catch signal and invoke graceful termination - stop := make(chan os.Signal, 1) - signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - <-stop - log.Print("[WARN] interrupt signal") - cancel() - }() - - app, err := New(opts) - if err != nil { - log.Fatalf("[ERROR] failed to setup application, %+v", err) - } - err = app.Run(ctx) - log.Printf("[INFO] remark terminated %s", err) -} - -// New prepares application and return it with all active parts -// doesn't start anything -func New(opts Opts) (*Application, error) { - - if err := makeDirs(opts.BackupLocation); err != nil { - return nil, err - } - - if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") { - return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL) - } - - storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites) - if err != nil { - return nil, err - } - - keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret) - if err != nil { - return nil, err - } - - dataService := &service.DataStore{ - Interface: storeEngine, - EditDuration: opts.EditDuration, - KeyStore: keyStore, - MaxCommentSize: opts.MaxCommentSize, - Admins: opts.Admins, - } - - loadingCache, err := makeCache(opts.Cache, opts.Mongo) - if err != nil { - return nil, err - } - - // token TTL is 5 minutes, inactivity interval 7+ days by default - jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) - - avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) - if err != nil { - return nil, errors.Wrap(err, "failed to make avatar store") - } - avatarProxy := &proxy.Avatar{ - Store: avatarStore, - RoutePath: "/api/v1/avatar", - RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), - } - - exporter := &migrator.Remark{DataStore: dataService} - - migr := &api.Migrator{ - Version: revision, - Cache: loadingCache, - NativeImporter: &migrator.Remark{DataStore: dataService}, - DisqusImporter: &migrator.Disqus{DataStore: dataService}, - WordPressImporter: &migrator.WordPress{DataStore: dataService}, - NativeExported: &migrator.Remark{DataStore: dataService}, - KeyStore: keyStore, - } - - authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) - imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} - commentFormatter := store.NewCommentFormatter(imgProxy) - - srv := &api.Rest{ - Version: revision, - DataService: dataService, - Exporter: exporter, - WebRoot: opts.WebRoot, - RemarkURL: opts.RemarkURL, - ImageProxy: imgProxy, - CommentFormatter: commentFormatter, - AvatarProxy: avatarProxy, - ReadOnlyAge: opts.ReadOnlyAge, - SharedSecret: opts.SharedSecret, - Authenticator: auth.Authenticator{ - JWTService: jwtService, - AdminEmail: opts.AdminEmail, - Providers: authProviders, - DevPasswd: opts.DevPasswd, - PermissionChecker: dataService, - }, - Cache: loadingCache, - } - - // no admin email, use admin@domain - if srv.Authenticator.AdminEmail == "" { - if u, err := url.Parse(opts.RemarkURL); err == nil { - srv.Authenticator.AdminEmail = "admin@" + u.Host - } - } - - srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore - - var devAuth *auth.DevAuthServer - if opts.Auth.Dev { - devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]} - } - - tch := make(chan struct{}) - return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService, - Opts: opts, terminated: tch}, nil -} - -// Run all application objects -func (a *Application) Run(ctx context.Context) error { - if a.DevPasswd != "" { - log.Printf("[WARN] running in dev mode") - } - - go func() { - // shutdown on context cancellation - <-ctx.Done() - a.restSrv.Shutdown() - a.migratorSrv.Shutdown() - if a.devAuth != nil { - a.devAuth.Shutdown() - } - if e := a.dataService.Close(); e != nil { - log.Printf("[WARN] failed to close store, %s", e) - } - - }() - a.activateBackup(ctx) // runs in goroutine for each site - go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only - if a.Auth.Dev { - go a.devAuth.Run() // dev oauth2 server on :8084 - } - a.restSrv.Run(a.Port) - close(a.terminated) - return nil -} - -// Wait for application completion (termination) -func (a *Application) Wait() { - <-a.terminated -} - -// activateBackup runs background backups for each site -func (a *Application) activateBackup(ctx context.Context) { - for _, siteID := range a.Sites { - backup := migrator.AutoBackup{ - Exporter: a.exporter, - BackupLocation: a.BackupLocation, - SiteID: siteID, - KeepMax: a.MaxBackupFiles, - Duration: 24 * time.Hour, - } - go backup.Do(ctx) - } -} - -// makeDataStore creates store for all sites -func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) { - switch group.Type { - case "bolt": - if err = makeDirs(group.Bolt.Path); err != nil { - return nil, errors.Wrap(err, "failed to create bolt store") - } - sites := []engine.BoltSite{} - for _, site := range siteNames { - sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)}) - } - result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...) - case "mongo": - mgServer, e := makeMongo(mg) - if e != nil { - return result, errors.Wrap(e, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, mg.DB, "") - result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) - default: - return nil, errors.Errorf("unsupported store type %s", group.Type) - } - return result, errors.Wrap(err, "can't initialize data store") -} - -func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { - switch group.Type { - case "fs": - if err := makeDirs(group.FS.Path); err != nil { - return nil, err - } - return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil - case "mongo": - mgServer, err := makeMongo(mg) + p.CommandHandler = func(command flags.Commander, args []string) error { + setupLog(opts.Dbg) + err := command.Execute(args) if err != nil { - return nil, errors.Wrap(err, "failed to create mongo server") + log.Printf("[ERROR] failed with %+v", err) } - conn := mongo.NewConnection(mgServer, mg.DB, "") - return avatar.NewGridFS(conn, group.RszLmt), nil - } - return nil, errors.Errorf("unsupported avatar store type %s", group.Type) -} - -func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) { - switch group.Type { - case "shared": - return keys.NewStaticStore(sharedSecret), nil - default: - return nil, errors.Errorf("unsupported key store type %s", group.Type) - } -} - -func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { - switch group.Type { - case "mem": - return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), - cache.MaxKeys(group.Max.Items)) - case "mongo": - mgServer, err := makeMongo(mg) - if err != nil { - return nil, errors.Wrap(err, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, mg.DB, "cache") - return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), - cache.MaxKeys(group.Max.Items)) - } - return nil, errors.Errorf("unsupported cache type %s", group.Type) -} - -// mkdir -p for all dirs -func makeDirs(dirs ...string) error { - - // exists returns whether the given file or directory exists or not - exists := func(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return true, err + return err } - for _, dir := range dirs { - ex, err := exists(dir) - if err != nil { - return errors.Wrapf(err, "can't check directory status for %s", dir) - } - if !ex { - if e := os.MkdirAll(dir, 0700); e != nil { - return errors.Wrapf(err, "can't make directory %s", dir) - } - } - } - return nil -} - -func makeMongo(mg MongoGroup) (result *mongo.Server, err error) { - if mg.URL == "" { - return nil, errors.New("no mongo URL provided") - } - return mongo.NewServerWithURL(mg.URL, 10*time.Second) -} - -func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider { - - makeParams := func(cid, secret string) auth.Params { - return auth.Params{ - JwtService: jwtService, - AvatarProxy: avatarProxy, - RemarkURL: opts.RemarkURL, - Cid: cid, - Csecret: secret, - PermissionChecker: ds, - } - } - - providers := []auth.Provider{} - if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" { - providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC))) - } - if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" { - providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC))) - } - if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" { - providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC))) - } - if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" { - providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC))) - } - if opts.Auth.Dev { - providers = append(providers, auth.NewDev(makeParams("", ""))) - } - - if len(providers) == 0 { - log.Printf("[WARN] no auth providers defined") - } - return providers -} - -// resetEnv clears all sensitive env vars -func resetEnv(envs ...string) { - for _, env := range envs { - if err := os.Unsetenv(env); err != nil { - log.Printf("[WARN] can't unset env %s, %s", env, err) + if _, err := p.Parse(); err != nil { + if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { + os.Exit(0) + } else { + os.Exit(1) } } } diff --git a/backend/app/main_test.go b/backend/app/main_test.go index 436d082f..0ffe37a7 100644 --- a/backend/app/main_test.go +++ b/backend/app/main_test.go @@ -1,227 +1,48 @@ package main import ( - "context" - "fmt" "io/ioutil" - "log" "net/http" "os" - "strings" + "sync" "syscall" "testing" "time" - "github.com/globalsign/mgo" - "github.com/go-pkgz/mongo" - "github.com/jessevdk/go-flags" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestApplication(t *testing.T) { - app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts { - o.Port = 18080 - return o - }) +func TestMain(t *testing.T) { - go func() { _ = app.Run(ctx) }() - time.Sleep(100 * time.Millisecond) // let server start - - // send ping - resp, err := http.Get("http://localhost:18080/api/v1/ping") - require.Nil(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, "pong", string(body)) - - // add comment - resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json", - strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`)) - require.Nil(t, err) - assert.Equal(t, http.StatusCreated, resp.StatusCode) - body, _ = ioutil.ReadAll(resp.Body) - t.Log(string(body)) - - assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email") - - app.Wait() -} - -func TestApplicationDevMode(t *testing.T) { - app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts { - o.Port = 18085 - o.DevPasswd = "password" - o.Auth.Dev = true - return o - }) - - go func() { _ = app.Run(ctx) }() - time.Sleep(100 * time.Millisecond) // let server start - - assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider") - assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider") - // send ping - resp, err := http.Get("http://localhost:18085/api/v1/ping") - require.Nil(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, "pong", string(body)) - - app.Wait() -} - -func TestApplicationWithMongo(t *testing.T) { - - mongoURL := os.Getenv("MONGO_TEST") - if mongoURL == "" { - mongoURL = "mongodb://localhost:27017/test" - } - if mongoURL == "skip" { - t.Skip("skip mongo app test") - } - - opts := Opts{} - // prepare options - p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com", - "--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) - require.Nil(t, err) - opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" - opts.BackupLocation = "/tmp" - - // create app - app, err := New(opts) - require.Nil(t, err) - - defer func() { - s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second) - assert.NoError(t, err) - conn := mongo.NewConnection(s, "test_remark", "") - _ = conn.WithDB(func(dbase *mgo.Database) error { - assert.NoError(t, dbase.DropDatabase()) - return nil - }) - }() - - ctx, cancel := context.WithCancel(context.Background()) - go func() { - time.Sleep(5 * time.Second) - log.Print("[TEST] terminate app") - cancel() - }() - go func() { _ = app.Run(ctx) }() - time.Sleep(100 * time.Millisecond) // let server start - - // send ping - resp, err := http.Get("http://localhost:12345/api/v1/ping") - require.Nil(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, "pong", string(body)) - - app.Wait() -} - -func TestApplicationFailed(t *testing.T) { - opts := Opts{} - p := flags.NewParser(&opts, flags.Default) - - // RO bolt location - _, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", - "--store.bolt.path=/dev/null"}) - assert.Nil(t, err) - _, err = New(opts) - assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ - "open /dev/null/remark.db: not a directory") - t.Log(err) - - // RO backup location - opts = Opts{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp", - "--backup=/dev/null/not-writable"}) - assert.Nil(t, err) - _, err = New(opts) - assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory") - t.Log(err) - - // invalid url - opts = Opts{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"}) - assert.Nil(t, err) - _, err = New(opts) - assert.EqualError(t, err, "invalid remark42 url demo.remark42.com") - t.Log(err) - - opts = Opts{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"}) - assert.NotNil(t, err, "blah is invalid type") - - opts.Store.Type = "blah" - _, err = New(opts) - assert.EqualError(t, err, "unsupported store type blah") - t.Log(err) -} - -func TestApplicationShutdown(t *testing.T) { - app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts { - o.Port = 18090 - return o - }) - st := time.Now() - err := app.Run(ctx) - assert.Nil(t, err) - assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") - app.Wait() -} - -func TestApplicationMainSignal(t *testing.T) { - os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", - "--port=18100", "--url=https://demo.remark42.com"} + os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", + "--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg"} go func() { - time.Sleep(100 * time.Millisecond) + time.Sleep(500 * time.Millisecond) err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM) require.Nil(t, err) }() - st := time.Now() - main() - assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") -} -func prepApp(t *testing.T, duration time.Duration, fn func(o Opts) Opts) (*Application, context.Context) { - opts := Opts{} - // prepare options - p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"}) - require.Nil(t, err) - opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp" - opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port) - opts.Store.Bolt.Timeout = 10 * time.Second - opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" - opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid" - opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid" - opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid" - opts.BackupLocation = "/tmp" - opts = fn(opts) - - os.Remove(opts.Store.Bolt.Path + "/remark.db") - - // create app - app, err := New(opts) - require.Nil(t, err) - - ctx, cancel := context.WithCancel(context.Background()) + wg := sync.WaitGroup{} + wg.Add(1) go func() { - time.Sleep(duration) - log.Print("[TEST] terminate app") - cancel() + st := time.Now() + main() + assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") + wg.Done() }() - return app, ctx + + time.Sleep(50 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:18202/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + wg.Wait() } diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 814a1323..c82caebf 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -60,7 +60,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment") return } - a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last")) + a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope)) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"id": id, "locator": locator}) } @@ -129,7 +129,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { } } - a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, "last")) + a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, lastCommentsScope)) render.Status(r, http.StatusOK) render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID}) } diff --git a/backend/app/rest/api/middleware.go b/backend/app/rest/api/middleware.go index 77185eb1..15ee1c7d 100644 --- a/backend/app/rest/api/middleware.go +++ b/backend/app/rest/api/middleware.go @@ -106,6 +106,7 @@ func Logger(ipFn func(ip string) string, flags ...LoggerFlag) func(http.Handler) if qun, err := url.QueryUnescape(q); err == nil { q = qun } + q = sanitizeQuery(q) remoteIP := strings.Split(r.RemoteAddr, ":")[0] if strings.HasPrefix(r.RemoteAddr, "[") { @@ -159,6 +160,27 @@ func getBodyAndUser(r *http.Request, flags []LoggerFlag) (body string, user stri return body, user } +func sanitizeQuery(u string) string { + out := []rune(u) + hide := []string{"password", "passwd", "secret", "credentials"} + for _, h := range hide { + if strings.Contains(strings.ToLower(u), h+"=") { + stPos := strings.Index(strings.ToLower(u), h+"=") + len(h) + 1 + fnPos := strings.Index(u[stPos:], "&") + if fnPos == -1 { + fnPos = len(u) + } else { + fnPos = stPos + fnPos + } + log.Print(stPos, fnPos) + for i := stPos; i < fnPos; i++ { + out[i] = rune('*') + } + } + } + return string(out) +} + func inLogFlags(f LoggerFlag, flags []LoggerFlag) bool { for _, flg := range flags { if (flg == LogAll && f != LogNone) || flg == f { diff --git a/backend/app/rest/api/middleware_test.go b/backend/app/rest/api/middleware_test.go index c2786057..a91f00ff 100644 --- a/backend/app/rest/api/middleware_test.go +++ b/backend/app/rest/api/middleware_test.go @@ -66,3 +66,19 @@ func TestMiddleware_GetBodyAndUser(t *testing.T) { assert.Equal(t, "", body) assert.Equal(t, ` - id1 "user1"`, user, "no user") } + +func TestMiddleware_sanitizeReqURL(t *testing.T) { + tbl := []struct { + in string + out string + }{ + {"", ""}, + {"/aa/bb?xyz=123", "/aa/bb?xyz=123"}, + {"/aa/bb?xyz=123&secret=asdfghjk", "/aa/bb?xyz=123&secret=********"}, + {"/aa/bb?xyz=123&secret=asdfghjk&key=val", "/aa/bb?xyz=123&secret=********&key=val"}, + {"/aa/bb?xyz=123&secret=asdfghjk&key=val&password=1234", "/aa/bb?xyz=123&secret=********&key=val&password=****"}, + } + for i, tt := range tbl { + assert.Equal(t, tt.out, sanitizeQuery(tt.in), "check #%d, %s", i, tt.in) + } +} diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 3010892a..51e5d331 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -145,7 +145,6 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/gzip") w.Header().Set("Content-Disposition", "attachment;filename="+exportFile) - w.WriteHeader(http.StatusOK) gzWriter := gzip.NewWriter(w) defer func() { if e := gzWriter.Close(); e != nil { @@ -154,6 +153,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { }() if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil { + log.Printf("[WARN] can't export, %+v", err) rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") return } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 57c18d2d..3c17dd71 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -60,6 +60,8 @@ type Rest struct { const hardBodyLimit = 1024 * 64 // limit size of body +const lastCommentsScope = "last" + type commentsWithInfo struct { Comments []store.Comment `json:"comments"` Info store.PostInfo `json:"info,omitempty"` @@ -123,7 +125,7 @@ func (s *Rest) routes() chi.Router { AllowedOrigins: []string{"*"}, AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"}, - ExposedHeaders: []string{"Link"}, + ExposedHeaders: []string{"Authorization"}, AllowCredentials: true, MaxAge: 300, }) diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 6f1edd0a..1a47a895 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -71,7 +71,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) { return } s.Cache.Flush(cache.Flusher(comment.Locator.SiteID). - Scopes(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)) + Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID)) render.Status(r, http.StatusCreated) render.JSON(w, r, &finalComment) @@ -120,7 +120,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { return } - s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last", user.ID)) + s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope, user.ID)) render.JSON(w, r, res) } diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index 6ee667bd..02460a68 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -114,23 +114,15 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) { limit = 0 } - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes("last") + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(lastCommentsScope) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, limit) if e != nil { return nil, e } comments = s.adminService.alterComments(comments, r) - // filter deleted from last comments view. Blocked marked as deleted and will sneak in without - filterDeleted := []store.Comment{} - for _, c := range comments { - if c.Deleted { - continue - } - filterDeleted = append(filterDeleted, c) - } - + filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted }) return encodeJSONWithHTML(filterDeleted) }) diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go index 31886ba1..a133cf1c 100644 --- a/backend/app/rest/api/rest_public_test.go +++ b/backend/app/rest/api/rest_public_test.go @@ -238,6 +238,7 @@ func TestRest_Last(t *testing.T) { err = json.Unmarshal([]byte(res), &comments) assert.Nil(t, err) assert.Equal(t, 2, len(comments), "should have 2 comments") + t.Logf("%+v", comments) } func TestRest_FindUserComments(t *testing.T) { diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 43a2f76c..cf1f7bc0 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -72,6 +72,21 @@ func TestRest_Shutdown(t *testing.T) { assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms") } +func TestRest_filterComments(t *testing.T) { + user := store.User{ID: "user1", Name: "user name 1"} + c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)} + c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)} + c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)} + + r := filterComments([]store.Comment{c1, c2, c3}, func(c store.Comment) bool { + return c.Text == "test test #1" || c.Text == "test test #3" + }) + assert.Equal(t, 2, len(r), "one comment filtered") +} + func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go index 39ef0fb6..da0562cf 100644 --- a/backend/app/rest/api/rss.go +++ b/backend/app/rest/api/rss.go @@ -67,7 +67,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss for site %s", siteID) - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last") + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope) data, err := s.Cache.Get(key, func() ([]byte, error) { comments, e := s.DataService.Last(siteID, maxRssItems) if e != nil { @@ -100,7 +100,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID) - key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, "last") + key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope) data, err := s.Cache.Get(key, func() (res []byte, e error) { comments, e := s.DataService.Last(siteID, maxLastCommentsReply) if e != nil { diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index e66c2131..a77446a0 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -135,22 +135,27 @@ func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) boo return false } + log.Printf("[DEBUG] dev user auth") s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) if len(s) != 2 { + log.Printf("[WARN] dev user auth failed, incorrect auth header %s", r.Header.Get("Authorization")) return false } b, err := base64.StdEncoding.DecodeString(s[1]) if err != nil { + log.Printf("[WARN] dev user auth failed, failed to decode %s, %s", s[1], err) return false } pair := strings.SplitN(string(b), ":", 2) if len(pair) != 2 { + log.Printf("[WARN] dev user auth failed, failed to split %s", string(b)) return false } if pair[0] != "dev" || pair[1] != a.DevPasswd { + log.Printf("[WARN] dev user auth failed, user/passwd mismatch %+v", pair) return false } diff --git a/backend/app/rest/proxy/avatar_test.go b/backend/app/rest/proxy/avatar_test.go index eb84bb75..53eba9a6 100644 --- a/backend/app/rest/proxy/avatar_test.go +++ b/backend/app/rest/proxy/avatar_test.go @@ -66,7 +66,7 @@ func TestAvatar_PutFailed(t *testing.T) { _, err := p.Put(u) assert.EqualError(t, err, "no picture for user1") - u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:12345/avater/pic"} + u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:22345/avater/pic"} _, err = p.Put(u) require.Error(t, err) assert.Contains(t, err.Error(), "connect: connection refused") diff --git a/backend/scripts/backup.sh b/backend/scripts/backup.sh new file mode 100644 index 00000000..4915df17 --- /dev/null +++ b/backend/scripts/backup.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +/srv/remark42 backup $@ diff --git a/backend/scripts/create-backup.sh b/backend/scripts/create-backup.sh deleted file mode 100755 index 38880440..00000000 --- a/backend/scripts/create-backup.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -# this scrips makes a backup file to /srv/var/userbackup--.gz -set -e -BACKUP_PATH=${BACKUP_PATH:-./var} -backup_file=${BACKUP_PATH}/userbackup-${1}-$(date +%s).gz -echo "make backup file for site $1 to $backup_file" -curl "http://127.0.0.1:8081/api/v1/admin/export?site=${1}&secret=${SECRET}" > ${backup_file} -echo "created backup ${backup_file}" \ No newline at end of file diff --git a/backend/scripts/import-disqus.sh b/backend/scripts/import-disqus.sh deleted file mode 100644 index 7b35e49a..00000000 --- a/backend/scripts/import-disqus.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -e -echo "import disqus file $1 to site $2" -curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=disqus&secret=${SECRET}" -echo "import completed" \ No newline at end of file diff --git a/backend/scripts/import-wordpress.sh b/backend/scripts/import-wordpress.sh deleted file mode 100755 index 8e60dabf..00000000 --- a/backend/scripts/import-wordpress.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -e -echo "import wordpress file $1 to site $2" -curl -X POST -H "Content-Type: application/xml" --data-binary @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=wordpress&secret=${SECRET}" -echo "import completed" \ No newline at end of file diff --git a/backend/scripts/import.sh b/backend/scripts/import.sh new file mode 100644 index 00000000..aa5d5492 --- /dev/null +++ b/backend/scripts/import.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +/srv/remark42 import $@ diff --git a/backend/scripts/migrate-data.sh b/backend/scripts/migrate-data.sh deleted file mode 100755 index 7397a0a4..00000000 --- a/backend/scripts/migrate-data.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# this scrips making a backup file to /tmp/export-remark.gz and loading it back -# useful to migrate data schema in case if new version of data store incomaptible with the stored comments. -set -e -echo "make backup file for site $1" -curl "http://127.0.0.1:8081/api/v1/admin/export?site=${1}&secret=${SECRET}" > /tmp/export-remark.gz - -BOLTDB_PATH=${BOLTDB_PATH:-./var} -BACKUP_PATH=${BACKUP_PATH:-./var} - -cp ${BOLTDB_PATH}/${1}.db ${BACKUP_PATH}/${1}-$(date +%s).db - -echo "import backup to site $1" -echo "unpack /tmp/export-remark.gz" -gunzip -c /tmp/export-remark.gz >/tmp/backup.remark -ls -laH /tmp/backup.remark - -echo "export to site $1" -curl -X POST -H "Content-Type: application/json" --data-binary @/tmp/backup.remark "http://127.0.0.1:8081/api/v1/admin/import?site=${1}&provider=native&secret=${SECRET}" - -rm /tmp/backup.remark -rm /tmp/export-remark.gz - -echo "migration completed" \ No newline at end of file diff --git a/backend/scripts/restore-backup.sh b/backend/scripts/restore-backup.sh deleted file mode 100755 index a50b0104..00000000 --- a/backend/scripts/restore-backup.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -set -e -echo "restore backup file $1 to site $2" -BACKUP_PATH=${BACKUP_PATH:-./var} -echo "unpack $1" -gunzip -c ${BACKUP_PATH}/$1 >/tmp/backup.remark - -echo "source file info" -ls -la /tmp/backup.remark - -curl -X POST -H "Content-Type: application/json" --data-binary @/tmp/backup.remark "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=native&secret=${SECRET}" -rm /tmp/backup.remark - -echo "backup restored" \ No newline at end of file diff --git a/backend/scripts/restore.sh b/backend/scripts/restore.sh new file mode 100644 index 00000000..b308abb1 --- /dev/null +++ b/backend/scripts/restore.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +/srv/remark42 restore $@ diff --git a/compose-dev-backend.yml b/compose-dev-backend.yml index 8af2392b..6ad99f6e 100644 --- a/compose-dev-backend.yml +++ b/compose-dev-backend.yml @@ -35,7 +35,6 @@ services: - "8084:8084" # local oauth2 server environment: - - USER # preset environment, UID on the host machine, i.e `id -u` - REMARK_URL=http://127.0.0.1:8080 - SECRET=12345 - STORE_BOLT_PATH=/srv/var/db @@ -46,5 +45,3 @@ services: - ADMIN=dev_user # set admin flag for default user on local ouath2 volumes: - ./var:/srv/var - - command: /srv/start.sh diff --git a/compose-dev-frontend.yml b/compose-dev-frontend.yml index 7d57674a..e1533c9c 100644 --- a/compose-dev-frontend.yml +++ b/compose-dev-frontend.yml @@ -28,7 +28,6 @@ services: - "8084:8084" # local oauth2 server environment: - - USER # preset environment, UID on the host machine, i.e `id -u` - REMARK_URL=http://127.0.0.1:8080 - SECRET=12345 - STORE_BOLT_PATH=/srv/var/db @@ -39,5 +38,3 @@ services: - ADMIN=dev_user # set admin flag for default user on local ouath2 volumes: - ./var:/srv/var - - command: /srv/start.sh diff --git a/docker-compose.yml b/docker-compose.yml index 9548e384..cce74474 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,6 @@ services: # - "80:8080" environment: - - USER - REMARK_URL - SECRET - STORE_BOLT_PATH=/srv/var/db @@ -36,5 +35,3 @@ services: # - DEV_PASSWD=password # development mode, be careful! volumes: - ./var:/srv/var - - command: /srv/start.sh diff --git a/start.sh b/entrypoint.sh similarity index 63% rename from start.sh rename to entrypoint.sh index c8a0c1d3..82867cb5 100755 --- a/start.sh +++ b/entrypoint.sh @@ -1,19 +1,11 @@ #!/bin/sh - echo "prepare environment" - # replace BASE_URL constant by REMARK_URL sed -i "s|https://demo.remark42.com|${REMARK_URL}|g" /srv/web/*.js # remove devtools attach helper. TODO: move to webpack loader sed -i "/REMOVE-START/,/REMOVE-END/d" /srv/web/iframe.html +chown -R app:app /srv/var 2>/dev/null + echo "start remark42 server" - -if [ -z "$USER" ] ; then \ - echo "No USER defined, runs under root!" - exec /srv/remark - -else - echo "runs under ${USER}" - /sbin/su-exec ${USER} /srv/remark -fi +/sbin/su-exec app /srv/remark42 $@ From dfa80d46e10ba04e9eb1516cfc76cfefd49ee313 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 26 Aug 2018 21:57:34 -0500 Subject: [PATCH 62/93] missed check for server.Execute status --- backend/app/cmd/server_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 5f25501e..f98a6fdf 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -196,7 +196,8 @@ func TestServerApp_MainSignal(t *testing.T) { "--port=18100", "--url=https://demo.remark42.com"} _, err := p.ParseArgs(args) require.Nil(t, err) - s.Execute(args) + err = s.Execute(args) + assert.NoError(t, err, "execute failed") assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") } From 8b3821e797f949856a83954248a1da77e0f122c7 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 30 Aug 2018 00:23:36 -0500 Subject: [PATCH 63/93] simplify cmds --- backend/app/cmd/backup.go | 13 +- backend/app/cmd/cmd.go | 11 ++ backend/app/cmd/import.go | 9 +- backend/app/cmd/server.go | 219 +++++++++++++++++---------------- backend/app/cmd/server_test.go | 34 ++--- backend/app/main.go | 2 +- 6 files changed, 150 insertions(+), 138 deletions(-) diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go index 815e7794..5e1a3984 100644 --- a/backend/app/cmd/backup.go +++ b/backend/app/cmd/backup.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -37,15 +36,15 @@ func (ec *BackupCommand) Execute(args []string) error { log.Printf("[DEBUG] export file %s", fname) - // prepare http client + // prepare http client and request client := http.Client{} + ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout) + defer cancel() exportURL := fmt.Sprintf("%s/api/v1/admin/export?site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret) req, err := http.NewRequest(http.MethodGet, exportURL, nil) if err != nil { return errors.Wrapf(err, "can't make export request for %s", exportURL) } - ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout) - defer cancel() // get with timeout resp, err := client.Do(req.WithContext(ctx)) @@ -59,11 +58,7 @@ func (ec *BackupCommand) Execute(args []string) error { }() if resp.StatusCode >= 300 { - body, e := ioutil.ReadAll(resp.Body) - if e != nil { - body = []byte("") - } - return errors.Errorf("error response %q, %s", resp.Status, body) + return responseError(resp) } fh, err := os.Create(fname) diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 15bee7c1..1414a356 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -4,7 +4,9 @@ package cmd import ( "bytes" + "io/ioutil" "log" + "net/http" "os" "path/filepath" "strings" @@ -64,3 +66,12 @@ func resetEnv(envs ...string) { } } } + +// responseError returns error with status and response body +func responseError(resp *http.Response) error { + body, e := ioutil.ReadAll(resp.Body) + if e != nil { + body = []byte("") + } + return errors.Errorf("error response %q, %s", resp.Status, body) +} diff --git a/backend/app/cmd/import.go b/backend/app/cmd/import.go index 13b4ef8b..f0ea5729 100644 --- a/backend/app/cmd/import.go +++ b/backend/app/cmd/import.go @@ -37,16 +37,15 @@ func (ic *ImportCommand) Execute(args []string) error { } client := http.Client{} + ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout) + defer cancel() importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s", ic.URL, ic.Site, ic.Provider, ic.SharedSecret) req, err := http.NewRequest(http.MethodPost, importURL, reader) if err != nil { return errors.Wrapf(err, "can't make import request for %s", importURL) } - ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout) - defer cancel() - - resp, err := client.Do(req.WithContext(ctx)) // closes reader + resp, err := client.Do(req.WithContext(ctx)) // closes request's reader if err != nil { return errors.Wrapf(err, "request failed for %s", importURL) } @@ -56,7 +55,7 @@ func (ic *ImportCommand) Execute(args []string) error { } }() if resp.StatusCode >= 300 { - return errors.Errorf("error response %s (%d)", resp.Status, resp.StatusCode) + return responseError(resp) } body, err := ioutil.ReadAll(resp.Body) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index c85c3889..1b56547e 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -27,8 +27,8 @@ import ( "github.com/umputun/remark/backend/app/store/service" ) -// ServerOpts with command line flags and env -type ServerOpts struct { +// ServerCommand with command line flags and env +type ServerCommand struct { RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` @@ -117,7 +117,7 @@ var Revision = "unknown" // serverApp holds all active objects type serverApp struct { - *ServerOpts + *ServerCommand restSrv *api.Rest migratorSrv *api.Migrator exporter migrator.Exporter @@ -127,7 +127,7 @@ type serverApp struct { } // Execute is the entry point for "server" command, called by flag parser -func (s *ServerOpts) Execute(args []string) error { +func (s *ServerCommand) Execute(args []string) error { log.Print("[INFO] start remark42 server") resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC") @@ -140,7 +140,7 @@ func (s *ServerOpts) Execute(args []string) error { cancel() }() - app, err := newServerApp(s) + app, err := s.newServerApp() if err != nil { log.Fatalf("[ERROR] failed to setup application, %+v", err) } @@ -154,50 +154,50 @@ func (s *ServerOpts) Execute(args []string) error { // newServerApp prepares application and return it with all active parts // doesn't start anything -func newServerApp(opts *ServerOpts) (*serverApp, error) { +func (s *ServerCommand) newServerApp() (*serverApp, error) { - if err := makeDirs(opts.BackupLocation); err != nil { + if err := s.makeDirs(s.BackupLocation); err != nil { return nil, err } - if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") { - return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL) + if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") { + return nil, errors.Errorf("invalid remark42 url %s", s.RemarkURL) } - storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites) + storeEngine, err := s.makeDataStore() if err != nil { return nil, err } - keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret) + keyStore, err := s.makeKeyStore() if err != nil { return nil, err } dataService := &service.DataStore{ Interface: storeEngine, - EditDuration: opts.EditDuration, + EditDuration: s.EditDuration, KeyStore: keyStore, - MaxCommentSize: opts.MaxCommentSize, - Admins: opts.Admins, + MaxCommentSize: s.MaxCommentSize, + Admins: s.Admins, } - loadingCache, err := makeCache(opts.Cache, opts.Mongo) + loadingCache, err := s.makeCache() if err != nil { return nil, err } // token TTL is 5 minutes, inactivity interval 7+ days by default - jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie) + jwtService := auth.NewJWT(keyStore, strings.HasPrefix(s.RemarkURL, "https://"), s.Auth.TTL.JWT, s.Auth.TTL.Cookie) - avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo) + avatarStore, err := s.makeAvatarStore() if err != nil { return nil, errors.Wrap(err, "failed to make avatar store") } avatarProxy := &proxy.Avatar{ Store: avatarStore, RoutePath: "/api/v1/avatar", - RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), + RemarkURL: strings.TrimSuffix(s.RemarkURL, "/"), } exporter := &migrator.Remark{DataStore: dataService} @@ -212,26 +212,26 @@ func newServerApp(opts *ServerOpts) (*serverApp, error) { KeyStore: keyStore, } - authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts) - imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL} + authProviders := s.makeAuthProviders(jwtService, avatarProxy, dataService) + imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL} commentFormatter := store.NewCommentFormatter(imgProxy) srv := &api.Rest{ Version: Revision, DataService: dataService, Exporter: exporter, - WebRoot: opts.WebRoot, - RemarkURL: opts.RemarkURL, + WebRoot: s.WebRoot, + RemarkURL: s.RemarkURL, ImageProxy: imgProxy, CommentFormatter: commentFormatter, AvatarProxy: avatarProxy, - ReadOnlyAge: opts.ReadOnlyAge, - SharedSecret: opts.SharedSecret, + ReadOnlyAge: s.ReadOnlyAge, + SharedSecret: s.SharedSecret, Authenticator: auth.Authenticator{ JWTService: jwtService, - AdminEmail: opts.AdminEmail, + AdminEmail: s.AdminEmail, Providers: authProviders, - DevPasswd: opts.DevPasswd, + DevPasswd: s.DevPasswd, PermissionChecker: dataService, }, Cache: loadingCache, @@ -239,25 +239,32 @@ func newServerApp(opts *ServerOpts) (*serverApp, error) { // no admin email, use admin@domain if srv.Authenticator.AdminEmail == "" { - if u, err := url.Parse(opts.RemarkURL); err == nil { + if u, err := url.Parse(s.RemarkURL); err == nil { srv.Authenticator.AdminEmail = "admin@" + u.Host } } - srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore + srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore var devAuth *auth.DevAuthServer - if opts.Auth.Dev { + if s.Auth.Dev { devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]} } - tch := make(chan struct{}) - return &serverApp{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService, - ServerOpts: opts, terminated: tch}, nil + return &serverApp{ + ServerCommand: s, + restSrv: srv, + migratorSrv: migr, + exporter: exporter, + devAuth: devAuth, + dataService: dataService, + terminated: make(chan struct{}), + }, nil } // Run all application objects func (a *serverApp) run(ctx context.Context) error { + log.Printf("%+v", a) if a.DevPasswd != "" { log.Printf("[WARN] running in dev mode") } @@ -305,76 +312,119 @@ func (a *serverApp) activateBackup(ctx context.Context) { } // makeDataStore creates store for all sites -func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) { - switch group.Type { +func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { + switch s.Store.Type { case "bolt": - if err = makeDirs(group.Bolt.Path); err != nil { + if err = s.makeDirs(s.Store.Bolt.Path); err != nil { return nil, errors.Wrap(err, "failed to create bolt store") } sites := []engine.BoltSite{} - for _, site := range siteNames { - sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)}) + for _, site := range s.Sites { + sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)}) } - result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...) + result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...) case "mongo": - mgServer, e := makeMongo(mg) + mgServer, e := s.makeMongo() if e != nil { return result, errors.Wrap(e, "failed to create mongo server") } - conn := mongo.NewConnection(mgServer, mg.DB, "") + conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) default: - return nil, errors.Errorf("unsupported store type %s", group.Type) + return nil, errors.Errorf("unsupported store type %s", s.Store.Type) } return result, errors.Wrap(err, "can't initialize data store") } -func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) { - switch group.Type { +func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { + switch s.Avatar.Type { case "fs": - if err := makeDirs(group.FS.Path); err != nil { + if err := s.makeDirs(s.Avatar.FS.Path); err != nil { return nil, err } - return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil + return avatar.NewLocalFS(s.Avatar.FS.Path, s.Avatar.RszLmt), nil case "mongo": - mgServer, err := makeMongo(mg) + mgServer, err := s.makeMongo() if err != nil { return nil, errors.Wrap(err, "failed to create mongo server") } - conn := mongo.NewConnection(mgServer, mg.DB, "") - return avatar.NewGridFS(conn, group.RszLmt), nil + conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") + return avatar.NewGridFS(conn, s.Avatar.RszLmt), nil } - return nil, errors.Errorf("unsupported avatar store type %s", group.Type) + return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type) } -func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) { - switch group.Type { +func (s *ServerCommand) makeKeyStore() (keys.Store, error) { + switch s.Key.Type { case "shared": - return keys.NewStaticStore(sharedSecret), nil + return keys.NewStaticStore(s.SharedSecret), nil default: - return nil, errors.Errorf("unsupported key store type %s", group.Type) + return nil, errors.Errorf("unsupported key store type %s", s.Key.Type) } } -func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) { - switch group.Type { +func (s *ServerCommand) makeCache() (cache.LoadingCache, error) { + switch s.Cache.Type { case "mem": - return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), - cache.MaxKeys(group.Max.Items)) + return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), + cache.MaxKeys(s.Cache.Max.Items)) case "mongo": - mgServer, err := makeMongo(mg) + mgServer, err := s.makeMongo() if err != nil { return nil, errors.Wrap(err, "failed to create mongo server") } - conn := mongo.NewConnection(mgServer, mg.DB, "cache") - return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value), - cache.MaxKeys(group.Max.Items)) + conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache") + return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), + cache.MaxKeys(s.Cache.Max.Items)) } - return nil, errors.Errorf("unsupported cache type %s", group.Type) + return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type) +} + +func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) { + if s.Mongo.URL == "" { + return nil, errors.New("no mongo URL provided") + } + return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second) +} + +func (s *ServerCommand) makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore) []auth.Provider { + + makeParams := func(cid, secret string) auth.Params { + return auth.Params{ + JwtService: jwtService, + AvatarProxy: avatarProxy, + RemarkURL: s.RemarkURL, + Cid: cid, + Csecret: secret, + PermissionChecker: ds, + } + } + + providers := []auth.Provider{} + if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" { + providers = append(providers, auth.NewGoogle(makeParams(s.Auth.Google.CID, s.Auth.Google.CSEC))) + } + if s.Auth.Github.CID != "" && s.Auth.Github.CSEC != "" { + providers = append(providers, auth.NewGithub(makeParams(s.Auth.Github.CID, s.Auth.Github.CSEC))) + } + if s.Auth.Facebook.CID != "" && s.Auth.Facebook.CSEC != "" { + providers = append(providers, auth.NewFacebook(makeParams(s.Auth.Facebook.CID, s.Auth.Facebook.CSEC))) + } + if s.Auth.Yandex.CID != "" && s.Auth.Yandex.CSEC != "" { + providers = append(providers, auth.NewYandex(makeParams(s.Auth.Yandex.CID, s.Auth.Yandex.CSEC))) + } + if s.Auth.Dev { + providers = append(providers, auth.NewDev(makeParams("", ""))) + } + + if len(providers) == 0 { + log.Printf("[WARN] no auth providers defined") + } + return providers } // mkdir -p for all dirs -func makeDirs(dirs ...string) error { +func (s *ServerCommand) makeDirs(dirs ...string) error { // exists returns whether the given file or directory exists or not exists := func(path string) (bool, error) { @@ -401,46 +451,3 @@ func makeDirs(dirs ...string) error { } return nil } - -func makeMongo(mg MongoGroup) (result *mongo.Server, err error) { - if mg.URL == "" { - return nil, errors.New("no mongo URL provided") - } - return mongo.NewServerWithURL(mg.URL, 10*time.Second) -} - -func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts *ServerOpts) []auth.Provider { - - makeParams := func(cid, secret string) auth.Params { - return auth.Params{ - JwtService: jwtService, - AvatarProxy: avatarProxy, - RemarkURL: opts.RemarkURL, - Cid: cid, - Csecret: secret, - PermissionChecker: ds, - } - } - - providers := []auth.Provider{} - if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" { - providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC))) - } - if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" { - providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC))) - } - if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" { - providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC))) - } - if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" { - providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC))) - } - if opts.Auth.Dev { - providers = append(providers, auth.NewDev(makeParams("", ""))) - } - - if len(providers) == 0 { - log.Printf("[WARN] no auth providers defined") - } - return providers -} diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index f98a6fdf..f0dc3943 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -20,7 +20,7 @@ import ( ) func TestServerApp(t *testing.T) { - app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand { o.Port = 18080 return o }) @@ -51,7 +51,7 @@ func TestServerApp(t *testing.T) { } func TestServerApp_DevMode(t *testing.T) { - app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand { o.Port = 18085 o.DevPasswd = "password" o.Auth.Dev = true @@ -85,7 +85,7 @@ func TestServerApp_WithMongo(t *testing.T) { t.Skip("skip mongo app test") } - opts := ServerOpts{} + opts := ServerCommand{} // prepare options p := flags.NewParser(&opts, flags.Default) _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com", @@ -95,7 +95,7 @@ func TestServerApp_WithMongo(t *testing.T) { opts.BackupLocation = "/tmp" // create app - app, err := newServerApp(&opts) + app, err := opts.newServerApp() require.Nil(t, err) defer func() { @@ -130,47 +130,47 @@ func TestServerApp_WithMongo(t *testing.T) { } func TestServerApp_Failed(t *testing.T) { - opts := ServerOpts{} + opts := ServerCommand{} p := flags.NewParser(&opts, flags.Default) // RO bolt location _, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.bolt.path=/dev/null"}) assert.Nil(t, err) - _, err = newServerApp(&opts) + _, err = opts.newServerApp() assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ "open /dev/null/remark.db: not a directory") t.Log(err) // RO backup location - opts = ServerOpts{} + opts = ServerCommand{} _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"}) assert.Nil(t, err) - _, err = newServerApp(&opts) + _, err = opts.newServerApp() assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory") t.Log(err) // invalid url - opts = ServerOpts{} + opts = ServerCommand{} _, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"}) assert.Nil(t, err) - _, err = newServerApp(&opts) + _, err = opts.newServerApp() assert.EqualError(t, err, "invalid remark42 url demo.remark42.com") t.Log(err) - opts = ServerOpts{} + opts = ServerCommand{} _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"}) assert.NotNil(t, err, "blah is invalid type") opts.Store.Type = "blah" - _, err = newServerApp(&opts) + _, err = opts.newServerApp() assert.EqualError(t, err, "unsupported store type blah") t.Log(err) } func TestServerApp_Shutdown(t *testing.T) { - app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts { + app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand { o.Port = 18090 return o }) @@ -190,7 +190,7 @@ func TestServerApp_MainSignal(t *testing.T) { }() st := time.Now() - s := ServerOpts{} + s := ServerCommand{} p := flags.NewParser(&s, flags.Default) args := []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", "--port=18100", "--url=https://demo.remark42.com"} @@ -201,8 +201,8 @@ func TestServerApp_MainSignal(t *testing.T) { assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec") } -func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerOpts) ServerOpts) (*serverApp, context.Context) { - opts := ServerOpts{} +func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) { + opts := ServerCommand{} // prepare options p := flags.NewParser(&opts, flags.Default) _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"}) @@ -220,7 +220,7 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerOpts) S os.Remove(opts.Store.Bolt.Path + "/remark.db") // create app - app, err := newServerApp(&opts) + app, err := opts.newServerApp() require.Nil(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/backend/app/main.go b/backend/app/main.go index df619011..5c0a598d 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -13,7 +13,7 @@ import ( // Opts has all commands type Opts struct { - ServerCmd cmd.ServerOpts `command:"server"` + ServerCmd cmd.ServerCommand `command:"server"` ImportCmd cmd.ImportCommand `command:"import"` BackupCmd cmd.BackupCommand `command:"backup"` RestoreCmd cmd.RestoreCommand `command:"restore"` From 78c27caa7d18ebdd87628385edd80f52cd9ccdd2 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 30 Aug 2018 01:53:02 -0500 Subject: [PATCH 64/93] remove dbg print --- backend/app/cmd/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 1b56547e..4d0999a8 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -264,7 +264,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { // Run all application objects func (a *serverApp) run(ctx context.Context) error { - log.Printf("%+v", a) if a.DevPasswd != "" { log.Printf("[WARN] running in dev mode") } From 0cd76dfd2f573a7c64402ab9b609b6b200fac056 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 2 Sep 2018 13:00:20 -0500 Subject: [PATCH 65/93] Feature/admin_store (#197) * generalize admins and email info with store interface * lint: missing group comment * fix admin group with shared substruct --- README.md | 4 +-- backend/app/cmd/cmd.go | 3 ++ backend/app/cmd/server.go | 45 ++++++++++++++++++-------- backend/app/cmd/server_test.go | 2 +- backend/app/main.go | 2 +- backend/app/rest/api/rest.go | 7 ++-- backend/app/rest/api/rest_public.go | 6 ++-- backend/app/rest/api/rest_test.go | 15 +++++---- backend/app/rest/auth/auth.go | 7 ++-- backend/app/rest/auth/provider.go | 2 +- backend/app/rest/auth/provider_test.go | 2 +- backend/app/store/admin/admin.go | 28 ++++++++++++++++ backend/app/store/admin/admin_test.go | 17 ++++++++++ backend/app/store/service/service.go | 24 ++++++++++---- 14 files changed, 123 insertions(+), 41 deletions(-) create mode 100644 backend/app/store/admin/admin.go create mode 100644 backend/app/store/admin/admin_test.go diff --git a/README.md b/README.md index b8756152..a84b3c48 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,8 @@ _this is the recommended way to run remark42_ | store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout | | store.mongo.url | STORE_MONGO_URL | | mongo url for data store | | store.mongo.db | STORE_MONGO_DB | | mongo db for data store | -| admin | ADMIN | | admin names (list of user ids), _multi_ | -| admin-email | ADMIN_EMAIL | `admin@${REMARK_URL}` | admin email | +| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ | +| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email | | backup | BACKUP_PATH | `./var/backup` | backups location | | max-back | MAX_BACKUP_FILES | `10` | max backup files to keep | | cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited | diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 1414a356..9407f2ed 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -16,6 +16,9 @@ import ( "github.com/pkg/errors" ) +// Revision sets from main +var Revision = "unknown" + // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format type fileParser struct { site string diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 4d0999a8..7f9f4314 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -14,6 +14,7 @@ import ( "github.com/coreos/bbolt" "github.com/go-pkgz/mongo" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/api" @@ -37,10 +38,9 @@ type ServerCommand struct { Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` + Admin AdminGroup `group:"admin" namespace:"admin" env-namespace:"ADMIN"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` - Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","` - AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"` DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"` BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"` MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` @@ -52,7 +52,6 @@ type ServerCommand struct { EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"` Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` - // Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` Auth struct { TTL struct { @@ -112,8 +111,14 @@ type KeyGroup struct { Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` } -// Revision sets from main -var Revision = "unknown" +// AdminGroup defines options group for admin params +type AdminGroup struct { + Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"` + Shared struct { + Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","` + Email string `long:"email" env:"EMAIL" default:"" description:"admin email"` + } `group:"shared" namespace:"shared" env-namespace:"SHARED"` +} // serverApp holds all active objects type serverApp struct { @@ -174,12 +179,17 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { return nil, err } + adminStore, err := s.makeAdminStore() + if err != nil { + return nil, err + } + dataService := &service.DataStore{ Interface: storeEngine, EditDuration: s.EditDuration, KeyStore: keyStore, + AdminStore: adminStore, MaxCommentSize: s.MaxCommentSize, - Admins: s.Admins, } loadingCache, err := s.makeCache() @@ -229,7 +239,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { SharedSecret: s.SharedSecret, Authenticator: auth.Authenticator{ JWTService: jwtService, - AdminEmail: s.AdminEmail, + AdminStore: adminStore, Providers: authProviders, DevPasswd: s.DevPasswd, PermissionChecker: dataService, @@ -237,13 +247,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { Cache: loadingCache, } - // no admin email, use admin@domain - if srv.Authenticator.AdminEmail == "" { - if u, err := url.Parse(s.RemarkURL); err == nil { - srv.Authenticator.AdminEmail = "admin@" + u.Host - } - } - srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore var devAuth *auth.DevAuthServer @@ -362,6 +365,20 @@ func (s *ServerCommand) makeKeyStore() (keys.Store, error) { } } +func (s *ServerCommand) makeAdminStore() (admin.Store, error) { + switch s.Admin.Type { + case "shared": + if s.Admin.Shared.Email == "" { // no admin email, use admin@domain + if u, err := url.Parse(s.RemarkURL); err == nil { + s.Admin.Shared.Email = "admin@" + u.Host + } + } + return admin.NewStaticStore(s.Admin.Shared.Admins, s.Admin.Shared.Email), nil + default: + return nil, errors.Errorf("unsupported admin store type %s", s.Key.Type) + } +} + func (s *ServerCommand) makeCache() (cache.LoadingCache, error) { switch s.Cache.Type { case "mem": diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index f0dc3943..1204f688 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -45,7 +45,7 @@ func TestServerApp(t *testing.T) { body, _ = ioutil.ReadAll(resp.Body) t.Log(string(body)) - assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email") + assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminStore.Email(""), "default admin email") app.Wait() } diff --git a/backend/app/main.go b/backend/app/main.go index 5c0a598d..70d3e9aa 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -11,7 +11,7 @@ import ( "github.com/umputun/remark/backend/app/cmd" ) -// Opts has all commands +// Opts with all cli commands and flags type Opts struct { ServerCmd cmd.ServerCommand `command:"server"` ImportCmd cmd.ImportCommand `command:"import"` diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 3c17dd71..6b2dcc4a 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -71,9 +71,10 @@ type commentsWithInfo struct { func (s *Rest) Run(port int) { log.Printf("[INFO] activate rest server on port %d", port) - if s.DataService != nil && len(s.DataService.Admins) > 0 { - log.Printf("[DEBUG] admins %+v", s.DataService.Admins) - } + // TODO: restore admin info dbg + //if s.DataService != nil && len(s.DataService.AdminStore.Admins()) > 0 { + // log.Printf("[DEBUG] admins %+v", s.DataService.Admins) + //} router := s.routes() diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index 02460a68..b6582acd 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -195,6 +195,8 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) { // GET /config?site=siteID - returns configuration func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site") + type config struct { Version string `json:"version"` EditDuration int `json:"edit_duration"` @@ -211,8 +213,8 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) { Version: s.Version, EditDuration: int(s.DataService.EditDuration.Seconds()), MaxCommentSize: s.DataService.MaxCommentSize, - Admins: s.DataService.Admins, - AdminEmail: s.Authenticator.AdminEmail, + Admins: s.DataService.AdminStore.Admins(siteID), + AdminEmail: s.DataService.AdminStore.Email(siteID), LowScore: s.ScoreThresholds.Low, CriticalScore: s.ScoreThresholds.Critical, ReadOnlyAge: s.ReadOnlyAge, diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index cf1f7bc0..8cac32e5 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -14,15 +14,16 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" + adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) @@ -90,20 +91,22 @@ func TestRest_filterComments(t *testing.T) { func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) + + adminStore := adminstore.NewStaticStore([]string{"a1", "a2"}, "admin@remark-42.com") + dataStore := &service.DataStore{ Interface: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, KeyStore: keys.NewStaticStore("123456"), - Admins: []string{"a1", "a2"}, + AdminStore: adminStore, } srv = &Rest{ DataService: dataStore, Authenticator: auth.Authenticator{ - DevPasswd: "password", - Providers: nil, - - AdminEmail: "admin@remark-42.com", + DevPasswd: "password", + Providers: nil, + AdminStore: adminStore, JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), }, Exporter: &migrator.Remark{DataStore: dataStore}, diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index a77446a0..e6ff1d56 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -9,13 +9,14 @@ import ( "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" ) // Authenticator is top level auth object providing middlewares type Authenticator struct { JWTService *JWT Providers []Provider - AdminEmail string + AdminStore admin.Store DevPasswd string PermissionChecker PermissionChecker } @@ -31,7 +32,7 @@ var devUser = store.User{ type PermissionChecker interface { IsVerified(siteID, userID string) bool IsBlocked(siteID, userID string) bool - IsAdmin(userID string) bool + IsAdmin(siteID, userID string) bool } // Auth middleware adds auth from session and populates user info @@ -98,7 +99,7 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { func (a *Authenticator) refreshExpiredToken(w http.ResponseWriter, claims *CustomClaims) (*CustomClaims, error) { if a.PermissionChecker != nil { - claims.User.Admin = a.PermissionChecker.IsAdmin(claims.User.ID) + claims.User.Admin = a.PermissionChecker.IsAdmin(claims.SiteID, claims.User.ID) claims.User.Blocked = a.PermissionChecker.IsBlocked(claims.SiteID, claims.User.ID) claims.User.Verified = a.PermissionChecker.IsVerified(claims.SiteID, claims.User.ID) } diff --git a/backend/app/rest/auth/provider.go b/backend/app/rest/auth/provider.go index 1fe0588d..4558384c 100644 --- a/backend/app/rest/auth/provider.go +++ b/backend/app/rest/auth/provider.go @@ -203,7 +203,7 @@ func (p Provider) setAvatar(u store.User) store.User { // setPermissions sets permission fields not handled by provider's MapUser, things like admin, verified and blocked func (p Provider) setPermissions(u store.User, siteID string) store.User { - u.Admin = p.PermissionChecker.IsAdmin(u.ID) + u.Admin = p.PermissionChecker.IsAdmin(siteID, u.ID) u.Verified = p.PermissionChecker.IsVerified(siteID, u.ID) u.Blocked = p.PermissionChecker.IsBlocked(siteID, u.ID) log.Printf("[DEBUG] set permissions for user %s, site %s - %+v", u.ID, siteID, u) diff --git a/backend/app/rest/auth/provider_test.go b/backend/app/rest/auth/provider_test.go index c26c9087..be78d789 100644 --- a/backend/app/rest/auth/provider_test.go +++ b/backend/app/rest/auth/provider_test.go @@ -233,6 +233,6 @@ type mockUserPermissions struct { blocked string } -func (m *mockUserPermissions) IsAdmin(userID string) bool { return userID == m.admin } +func (m *mockUserPermissions) IsAdmin(siteID, userID string) bool { return userID == m.admin } func (m *mockUserPermissions) IsVerified(siteID, userID string) bool { return userID == m.verified } func (m *mockUserPermissions) IsBlocked(siteID, userID string) bool { return userID == m.blocked } diff --git a/backend/app/store/admin/admin.go b/backend/app/store/admin/admin.go new file mode 100644 index 00000000..3c034972 --- /dev/null +++ b/backend/app/store/admin/admin.go @@ -0,0 +1,28 @@ +package admin + +// Store defines interface returning admins info for given site +type Store interface { + Admins(siteID string) (ids []string) + Email(siteID string) (email string) +} + +// StaticStore implements keys.Store with a single, predefined key +type StaticStore struct { + admins []string + email string +} + +// NewStaticStore makes StaticStore instance with given key +func NewStaticStore(admins []string, email string) *StaticStore { + return &StaticStore{admins: admins, email: email} +} + +// Admins returns static list of admin's ids, the same for all sites +func (s *StaticStore) Admins(string) (ids []string) { + return s.admins +} + +// Email gets static email address +func (s *StaticStore) Email(string) (email string) { + return s.email +} diff --git a/backend/app/store/admin/admin_test.go b/backend/app/store/admin/admin_test.go new file mode 100644 index 00000000..acd4df2c --- /dev/null +++ b/backend/app/store/admin/admin_test.go @@ -0,0 +1,17 @@ +package admin + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStaticStore_Get(t *testing.T) { + var ks Store = NewStaticStore([]string{"123", "xyz"}, "aa@example.com") + + a := ks.Admins("any") + assert.Equal(t, []string{"123", "xyz"}, a) + + email := ks.Email("blah") + assert.Equal(t, "aa@example.com", email) +} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index c9c35f03..c9f2d852 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" @@ -17,8 +18,8 @@ type DataStore struct { engine.Interface EditDuration time.Duration KeyStore keys.Store + AdminStore admin.Store MaxCommentSize int - Admins []string // granular locks scopedLocks struct { @@ -32,6 +33,16 @@ const defaultCommentMaxSize = 2000 // Create prepares comment and forward to Interface.Create func (s *DataStore) Create(comment store.Comment) (commentID string, err error) { + + if comment, err = s.prepareNewComment(comment); err != nil { + return "", errors.Wrap(err, "failed to prepare comment") + } + + return s.Interface.Create(comment) +} + +// prepareNewComment sets new comment fields, hashing and sanitizing data +func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, error) { // fill ID and time if empty if comment.ID == "" { comment.ID = uuid.New().String() @@ -47,11 +58,10 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) secret, err := s.KeyStore.Get(comment.Locator.SiteID) if err != nil { - return "", errors.Wrapf(err, "can't get secret for site %s", comment.Locator.SiteID) + return store.Comment{}, errors.Wrapf(err, "can't get secret for site %s", comment.Locator.SiteID) } comment.User.HashIP(secret) // replace ip by hash - - return s.Interface.Create(comment) + return comment, nil } // SetPin pin/un-pin comment as special @@ -170,9 +180,9 @@ func (s *DataStore) ValidateComment(c *store.Comment) error { } // IsAdmin checks if usesID in the list of admins -func (s *DataStore) IsAdmin(userID string) bool { - for _, admin := range s.Admins { - if admin == userID { +func (s *DataStore) IsAdmin(siteID string, userID string) bool { + for _, a := range s.AdminStore.Admins(siteID) { + if a == userID { return true } } From 3cc6762ebf4f578b6fa5c8b44775322857a4f607 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 2 Sep 2018 23:14:29 -0500 Subject: [PATCH 66/93] add comments --- backend/app/cmd/backup.go | 2 +- backend/app/cmd/cmd.go | 1 + backend/app/cmd/server.go | 2 +- backend/app/rest/auth/auth.go | 23 ++++++++++++----------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go index 5e1a3984..7d3047df 100644 --- a/backend/app/cmd/backup.go +++ b/backend/app/cmd/backup.go @@ -13,7 +13,7 @@ import ( ) // BackupCommand set of flags and command for export -// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. /) BACKUP_PATH ignored. +// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored. type BackupCommand struct { ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"` diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 9407f2ed..cb1d2c2f 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -29,6 +29,7 @@ type fileParser struct { // parse apply template and also concat path and file. In case if file contains path separator path will be ignored func (p *fileParser) parse(now time.Time) (string, error) { + // file/location paramaters my have template masks fileTemplate := struct { YYYYMMDD string YYYY string diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 7f9f4314..249f52de 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -14,7 +14,6 @@ import ( "github.com/coreos/bbolt" "github.com/go-pkgz/mongo" "github.com/pkg/errors" - "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest/api" @@ -22,6 +21,7 @@ import ( "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/keys" diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index e6ff1d56..e71ce422 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -28,7 +28,7 @@ var devUser = store.User{ Admin: true, } -// PermissionChecker defines interface to get user flags +// PermissionChecker defines interface to check user flags type PermissionChecker interface { IsVerified(siteID, userID string) bool IsBlocked(siteID, userID string) bool @@ -41,7 +41,7 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { f := func(h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - if a.basicDevUser(w, r) { // fail-back to dev user if enabled + if a.basicDevUser(w, r) { // use dev user basic auth if enabled user := devUser r = rest.SetUserInfo(r, user) h.ServeHTTP(w, r) @@ -49,13 +49,13 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { } claims, err := a.JWTService.Get(r) - if err != nil && reqAuth { // in full auth lack of session causes Unauthorized - log.Printf("[DEBUG] failed auth, %s", err) - http.Error(w, "Unauthorized", http.StatusUnauthorized) - return - } - - if err != nil { // in anonymous mode just pass it to the next handler + if err != nil { + if reqAuth { // in full auth lack of token causes Unauthorized + log.Printf("[DEBUG] failed auth, %s", err) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + // in anonymous mode just pass it to the next handler h.ServeHTTP(w, r) return } @@ -74,7 +74,7 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { return } - if a.JWTService.HasFlags(claims) { + if a.JWTService.HasFlags(claims) { // flags in token indicate special use cases, not for login log.Printf("[DEBUG] invalid token flags for %s/%s", claims.User.Name, claims.User.ID) http.Error(w, "Unauthorized", http.StatusUnauthorized) return @@ -97,6 +97,7 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { return f } +// refreshExpiredToken makes new token with passed claims, but only if permission allowed func (a *Authenticator) refreshExpiredToken(w http.ResponseWriter, claims *CustomClaims) (*CustomClaims, error) { if a.PermissionChecker != nil { claims.User.Admin = a.PermissionChecker.IsAdmin(claims.SiteID, claims.User.ID) @@ -110,7 +111,7 @@ func (a *Authenticator) refreshExpiredToken(w http.ResponseWriter, claims *Custo return claims, nil } -// AdminOnly allows access to admins +// AdminOnly middleware allows access for admins only func (a *Authenticator) AdminOnly(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { From 84e777035932d8d840a7c5b00b78056dd7051fb2 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 3 Sep 2018 23:13:29 -0500 Subject: [PATCH 67/93] Feature/admin-migrator (#198) * auth with a secret as admin * get rid of separate migrator service * clean sanitize dbg prints * fix dbg logging and set migrator routes --- Makefile | 2 +- backend/app/cmd/backup.go | 4 +- backend/app/cmd/import.go | 2 +- backend/app/cmd/server.go | 8 +- backend/app/rest/api/admin.go | 33 +------- backend/app/rest/api/admin_test.go | 1 - backend/app/rest/api/middleware.go | 1 - backend/app/rest/api/migrator.go | 113 +++++--------------------- backend/app/rest/api/migrator_test.go | 58 ++++++++----- backend/app/rest/api/rest.go | 7 +- backend/app/rest/api/rest_test.go | 9 +- backend/app/rest/auth/auth.go | 38 ++++++++- backend/app/rest/auth/auth_test.go | 18 ++++ 13 files changed, 133 insertions(+), 161 deletions(-) diff --git a/Makefile b/Makefile index 27a983cf..fa3b1872 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,4 @@ deploy: docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip docker rm -f remark42.bin - +.PHONY: bin \ No newline at end of file diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go index 7d3047df..4307e474 100644 --- a/backend/app/cmd/backup.go +++ b/backend/app/cmd/backup.go @@ -20,7 +20,7 @@ type BackupCommand struct { Site string `long:"site" env:"SITE" default:"remark" description:"site name"` SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` - URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` + URL string `long:"url" default:"http://127.0.0.1:8080" description:"base url"` } // Execute runs export with ExportCommand parameters, entry point for "export" command @@ -40,7 +40,7 @@ func (ec *BackupCommand) Execute(args []string) error { client := http.Client{} ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout) defer cancel() - exportURL := fmt.Sprintf("%s/api/v1/admin/export?site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret) + exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret) req, err := http.NewRequest(http.MethodGet, exportURL, nil) if err != nil { return errors.Wrapf(err, "can't make export request for %s", exportURL) diff --git a/backend/app/cmd/import.go b/backend/app/cmd/import.go index f0ea5729..5e5b3d42 100644 --- a/backend/app/cmd/import.go +++ b/backend/app/cmd/import.go @@ -23,7 +23,7 @@ type ImportCommand struct { SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` - URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` + URL string `long:"url" default:"http://127.0.0.1:8080" description:"migrator base url"` } // Execute runs import with ImportCommand parameters, entry point for "import" command diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 249f52de..79c42623 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -213,7 +213,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { exporter := &migrator.Remark{DataStore: dataService} migr := &api.Migrator{ - Version: Revision, Cache: loadingCache, NativeImporter: &migrator.Remark{DataStore: dataService}, DisqusImporter: &migrator.Disqus{DataStore: dataService}, @@ -229,12 +228,12 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { srv := &api.Rest{ Version: Revision, DataService: dataService, - Exporter: exporter, WebRoot: s.WebRoot, RemarkURL: s.RemarkURL, ImageProxy: imgProxy, CommentFormatter: commentFormatter, AvatarProxy: avatarProxy, + Migrator: migr, ReadOnlyAge: s.ReadOnlyAge, SharedSecret: s.SharedSecret, Authenticator: auth.Authenticator{ @@ -243,6 +242,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { Providers: authProviders, DevPasswd: s.DevPasswd, PermissionChecker: dataService, + KeyStore: keyStore, }, Cache: loadingCache, } @@ -275,7 +275,6 @@ func (a *serverApp) run(ctx context.Context) error { // shutdown on context cancellation <-ctx.Done() a.restSrv.Shutdown() - a.migratorSrv.Shutdown() if a.devAuth != nil { a.devAuth.Shutdown() } @@ -284,8 +283,7 @@ func (a *serverApp) run(ctx context.Context) error { } }() - a.activateBackup(ctx) // runs in goroutine for each site - go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only + a.activateBackup(ctx) // runs in goroutine for each site if a.Auth.Dev { go a.devAuth.Run() // dev oauth2 server on :8084 } diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index c82caebf..0dd3d5b3 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -1,10 +1,7 @@ package api import ( - "compress/gzip" "errors" - "fmt" - "io" "log" "net/http" "path" @@ -30,6 +27,7 @@ type admin struct { authenticator auth.Authenticator readOnlyAge int avatarProxy *proxy.Avatar + migrator *Migrator } func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router { @@ -41,10 +39,12 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route router.Get("/user/{userid}", a.getUserInfoCtrl) router.Get("/deleteme", a.deleteMeRequestCtrl) router.Put("/verify/{userid}", a.setVerifyCtrl) - router.Get("/export", a.exportCtrl) router.Put("/pin/{id}", a.setPinCtrl) router.Get("/blocked", a.blockedUsersCtrl) router.Put("/readonly", a.setReadOnlyCtrl) + + a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import + return router } @@ -221,31 +221,6 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"id": commentID, "locator": locator, "pin": pinStatus}) } -// GET /export?site=site-id?mode=file|stream -// exports all comments for siteID as json stream or gz file -func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { - siteID := r.URL.Query().Get("site") - var writer io.Writer = w - if r.URL.Query().Get("mode") == "file" { - exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102")) - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Disposition", "attachment;filename="+exportFile) - w.WriteHeader(http.StatusOK) - gzWriter := gzip.NewWriter(w) - defer func() { - if e := gzWriter.Close(); e != nil { - log.Printf("[WARN] can't close gzip writer, %s", e) - } - }() - writer = gzWriter - } - - if _, err := a.exporter.Export(writer, siteID); err != nil { - rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") - return - } -} - func (a *admin) checkBlocked(siteID string, user store.User) bool { return a.dataService.IsBlocked(siteID, user.ID) } diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index a5805783..567256d9 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -413,7 +413,6 @@ func TestAdmin_Verify(t *testing.T) { assert.Equal(t, 2, len(comments.Comments), "should have 2 comments") assert.Equal(t, "test test #1", comments.Comments[0].Text) assert.False(t, comments.Comments[0].User.Verified) - } func TestAdmin_ExportStream(t *testing.T) { diff --git a/backend/app/rest/api/middleware.go b/backend/app/rest/api/middleware.go index 15ee1c7d..f73f48ab 100644 --- a/backend/app/rest/api/middleware.go +++ b/backend/app/rest/api/middleware.go @@ -172,7 +172,6 @@ func sanitizeQuery(u string) string { } else { fnPos = stPos + fnPos } - log.Print(stPos, fnPos) for i := stPos; i < fnPos; i++ { out[i] = rune('*') } diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 51e5d331..01534e4d 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -2,18 +2,13 @@ package api import ( "compress/gzip" - "context" "fmt" + "io" "log" "net/http" - "strings" - "sync" "time" - "github.com/didip/tollbooth" - "github.com/didip/tollbooth_chi" "github.com/go-chi/chi" - "github.com/go-chi/chi/middleware" "github.com/go-chi/render" "github.com/umputun/remark/backend/app/migrator" @@ -22,59 +17,19 @@ import ( "github.com/umputun/remark/backend/app/store/keys" ) -// Migrator rest runs on unexposed port and available for local requests only +// Migrator rest with import and export controllers type Migrator struct { - Version string Cache cache.LoadingCache NativeImporter migrator.Importer DisqusImporter migrator.Importer WordPressImporter migrator.Importer NativeExported migrator.Exporter KeyStore keys.Store - - httpServer *http.Server - lock sync.Mutex } -// Run the listener and request's router, activate rest server -// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way -func (m *Migrator) Run(port int) { - log.Printf("[INFO] activate import server on port %d", port) - router := m.routes() - - m.lock.Lock() - m.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router} - m.lock.Unlock() - - err := m.httpServer.ListenAndServe() - log.Printf("[WARN] http server terminated, %s", err) -} - -// Shutdown import http server -func (m *Migrator) Shutdown() { - log.Print("[WARN] shutdown import server") - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - m.lock.Lock() - if m.httpServer != nil { - if err := m.httpServer.Shutdown(ctx); err != nil { - log.Printf("[DEBUG] importer shutdown error, %s", err) - } - } - m.lock.Unlock() - - log.Print("[DEBUG] shutdown import server completed") -} - -func (m *Migrator) routes() chi.Router { - router := chi.NewRouter() - router.Use(middleware.RealIP, Recoverer) - router.Use(middleware.Throttle(1000), middleware.Timeout(15*time.Minute)) - router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil))) - router.Use(AppInfo("remark42-migrator", m.Version), Ping, Logger(nil, LogAll)) - router.Post("/api/v1/admin/import", m.importCtrl) - router.Get("/api/v1/admin/export", m.exportCtrl) +func (m *Migrator) withRoutes(router chi.Router) chi.Router { + router.Get("/export", m.exportCtrl) + router.Post("/import", m.importCtrl) return router } @@ -83,20 +38,6 @@ func (m *Migrator) routes() chi.Router { func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") - secret := r.URL.Query().Get("secret") - - skey, err := m.KeyStore.Get(siteID) - if err != nil { - render.Status(r, http.StatusForbidden) - render.JSON(w, r, JSON{"status": "error", "details": "secret key store"}) - return - } - - if strings.TrimSpace(secret) == "" || secret != skey { - render.Status(r, http.StatusForbidden) - render.JSON(w, r, JSON{"status": "error", "details": "secret key"}) - return - } var importer migrator.Importer switch r.URL.Query().Get("provider") { @@ -108,7 +49,7 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { importer = m.NativeImporter } - log.Printf("[DEBUG] import request for site=%s", siteID) + log.Printf("[DEBUG] import request for site=%s, provider=%s", siteID, r.URL.Query().Get("provider")) size, err := importer.Import(r.Body, siteID) if err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") @@ -120,40 +61,28 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"status": "ok", "size": size}) } -// GET /export?site=site-id&secret=12345 +// GET /export?site=site-id&secret=12345&?mode=file|stream // exports all comments for siteID as gz file func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") - secret := r.URL.Query().Get("secret") - skey, err := m.KeyStore.Get(siteID) - if err != nil { - render.Status(r, http.StatusForbidden) - render.JSON(w, r, JSON{"status": "error", "details": "secret key store"}) - return + var writer io.Writer = w + if r.URL.Query().Get("mode") == "file" { + exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102")) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Disposition", "attachment;filename="+exportFile) + w.WriteHeader(http.StatusOK) + gzWriter := gzip.NewWriter(w) + defer func() { + if e := gzWriter.Close(); e != nil { + log.Printf("[WARN] can't close gzip writer, %s", e) + } + }() + writer = gzWriter } - if strings.TrimSpace(secret) == "" || secret != skey { - render.Status(r, http.StatusForbidden) - render.JSON(w, r, JSON{"status": "error", "details": "secret key"}) - return - } - - exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102")) - log.Printf("[DEBUG] import request for site=%s to %s", siteID, exportFile) - - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Disposition", "attachment;filename="+exportFile) - gzWriter := gzip.NewWriter(w) - defer func() { - if e := gzWriter.Close(); e != nil { - log.Printf("[WARN] can't close gzip writer, %s", e) - } - }() - - if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil { - log.Printf("[WARN] can't export, %+v", err) + if _, err := m.NativeExported.Export(writer, siteID); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") return } diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 5b62721a..eb0c2b52 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -12,12 +12,15 @@ import ( "time" "github.com/coreos/bbolt" + "github.com/go-chi/chi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/migrator" + "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" "github.com/umputun/remark/backend/app/store" + adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" @@ -32,7 +35,7 @@ func TestMigrator_Import(t *testing.T) { {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} - req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r) + req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r) assert.Nil(t, err) resp, err := client.Do(req) assert.Nil(t, err) @@ -51,7 +54,7 @@ func TestMigrator_ImportFromWP(t *testing.T) { r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1)) client := &http.Client{Timeout: 1 * time.Second} - req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress&secret=123456", r) + req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress&secret=123456", r) assert.Nil(t, err) req.Header.Add("Content-Type", "application/xml; charset=utf-8") resp, err := client.Do(req) @@ -88,11 +91,11 @@ func TestMigrator_ImportRejected(t *testing.T) { {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} - req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r) + req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r) assert.Nil(t, err) resp, err := client.Do(req) assert.Nil(t, err) - assert.Equal(t, http.StatusForbidden, resp.StatusCode) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) } func TestMigrator_Export(t *testing.T) { @@ -104,13 +107,14 @@ func TestMigrator_Export(t *testing.T) { {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} - req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r) + req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r) require.Nil(t, err) resp, err := client.Do(req) require.Nil(t, err) require.Equal(t, http.StatusCreated, resp.StatusCode) - req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=123456", nil) + // check file mode + req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t&secret=123456", nil) require.Nil(t, err) resp, err = client.Do(req) require.Nil(t, err) @@ -125,44 +129,54 @@ func TestMigrator_Export(t *testing.T) { assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\"")) t.Logf("%s", string(ungzBody)) - req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=bad", nil) + // check stream mode + req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t&secret=123456", nil) require.Nil(t, err) resp, err = client.Do(req) require.Nil(t, err) - require.Equal(t, 403, resp.StatusCode) -} + require.Equal(t, 200, resp.StatusCode) + require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type")) -func TestMigrator_Shutdown(t *testing.T) { - srv := Migrator{} - go func() { - time.Sleep(100 * time.Millisecond) - srv.Shutdown() - }() - st := time.Now() - srv.Run(0) - assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms") + body, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Equal(t, 2, strings.Count(string(body), "\n")) + assert.Equal(t, 2, strings.Count(string(body), "\"text\"")) + t.Logf("%s", string(body)) + + req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t&secret=bad", nil) + require.Nil(t, err) + resp, err = client.Do(req) + require.Nil(t, err) + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) } func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) - dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("123456")} + ks := keys.NewStaticStore("123456") + dataStore := &service.DataStore{Interface: b, KeyStore: ks} svc = &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, NativeImporter: &migrator.Remark{DataStore: dataStore}, NativeExported: &migrator.Remark{DataStore: dataStore}, Cache: &cache.Nop{}, - KeyStore: keys.NewStaticStore("123456"), + KeyStore: ks, } - routes := svc.routes() + a := auth.Authenticator{ + DevPasswd: "password", + Providers: nil, + AdminStore: adminstore.NewStaticStore([]string{"a1", "a2"}, "admin@remark-42.com"), + JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), + KeyStore: ks, + } + routes := svc.withRoutes(chi.NewRouter().With(a.Auth(true)).With(a.AdminOnly)) ts = httptest.NewServer(routes) return svc, dataStore, ts } func cleanupImportSrv(m *Migrator, ts *httptest.Server) { ts.Close() - m.Shutdown() os.Remove(testDb) } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 6b2dcc4a..fc3be7ee 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -22,7 +22,6 @@ import ( "github.com/pkg/errors" "github.com/rakyll/statik/fs" - "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" @@ -37,11 +36,11 @@ type Rest struct { DataService *service.DataStore Authenticator auth.Authenticator - Exporter migrator.Exporter Cache cache.LoadingCache AvatarProxy *proxy.Avatar ImageProxy *proxy.Image CommentFormatter *store.CommentFormatter + Migrator *Migrator WebRoot string RemarkURL string @@ -115,7 +114,7 @@ func (s *Rest) routes() chi.Router { s.adminService = admin{ dataService: s.DataService, - exporter: s.Exporter, + migrator: s.Migrator, cache: s.Cache, authenticator: s.Authenticator, readOnlyAge: s.ReadOnlyAge, @@ -187,7 +186,7 @@ func (s *Rest) routes() chi.Router { rauth.Post("/deleteme", s.deleteMeCtrl) // admin routes, admin users only - rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly, Logger(nil, LogAll))) + rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly)) }) }) diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 8cac32e5..df27dacf 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -109,7 +109,6 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { AdminStore: adminStore, JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), }, - Exporter: &migrator.Remark{DataStore: dataStore}, Cache: &cache.Nop{}, WebRoot: "/tmp", RemarkURL: "https://demo.remark42.com", @@ -117,6 +116,14 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { ImageProxy: &proxy.Image{}, ReadOnlyAge: 10, CommentFormatter: store.NewCommentFormatter(&proxy.Image{}), + Migrator: &Migrator{ + DisqusImporter: &migrator.Disqus{DataStore: dataStore}, + WordPressImporter: &migrator.WordPress{DataStore: dataStore}, + NativeImporter: &migrator.Remark{DataStore: dataStore}, + NativeExported: &migrator.Remark{DataStore: dataStore}, + Cache: &cache.Nop{}, + KeyStore: keys.NewStaticStore("123456"), + }, } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10 diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index e71ce422..5e62b1e6 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -10,6 +10,7 @@ import ( "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" + "github.com/umputun/remark/backend/app/store/keys" ) // Authenticator is top level auth object providing middlewares @@ -17,6 +18,7 @@ type Authenticator struct { JWTService *JWT Providers []Provider AdminStore admin.Store + KeyStore keys.Store DevPasswd string PermissionChecker PermissionChecker } @@ -28,6 +30,13 @@ var devUser = store.User{ Admin: true, } +var adminUser = store.User{ + ID: "admin", + Name: "admin", + Picture: "/api/v1/avatar/remark.image", + Admin: true, +} + // PermissionChecker defines interface to check user flags type PermissionChecker interface { IsVerified(siteID, userID string) bool @@ -41,9 +50,15 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { f := func(h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { + // if secret key matches and site in request return admin + if a.checkSecretKey(r) { + r = rest.SetUserInfo(r, adminUser) + h.ServeHTTP(w, r) + return + } + if a.basicDevUser(w, r) { // use dev user basic auth if enabled - user := devUser - r = rest.SetUserInfo(r, user) + r = rest.SetUserInfo(r, devUser) h.ServeHTTP(w, r) return } @@ -97,6 +112,25 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { return f } +func (a *Authenticator) checkSecretKey(r *http.Request) bool { + if a.KeyStore == nil { + return false + } + + siteID := r.URL.Query().Get("site") + secret := r.URL.Query().Get("secret") + + skey, err := a.KeyStore.Get(siteID) + if err != nil { + return false + } + + if strings.TrimSpace(secret) == "" || secret != skey { + return false + } + return true +} + // refreshExpiredToken makes new token with passed claims, but only if permission allowed func (a *Authenticator) refreshExpiredToken(w http.ResponseWriter, claims *CustomClaims) (*CustomClaims, error) { if a.PermissionChecker != nil { diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index 33dc0327..4efc80f9 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -212,6 +212,24 @@ func TestAdminRequired(t *testing.T) { } +func TestAuthWithSecret(t *testing.T) { + a := Authenticator{DevPasswd: "123456", KeyStore: keys.NewStaticStore("secretkey")} + router := chi.NewRouter() + router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(201) + }) + server := httptest.NewServer(router) + defer server.Close() + + resp, err := http.Get(server.URL + "/auth?secret=secretkey") + require.NoError(t, err) + assert.Equal(t, 201, resp.StatusCode, "valid auth user with secret, admin") + + resp, err = http.Get(server.URL + "/auth?secret=badsecret") + require.NoError(t, err) + assert.Equal(t, 401, resp.StatusCode, "invalid auth with bad secret") +} + func withBasicAuth(r *http.Request, username, password string) *http.Request { auth := username + ":" + password r.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth))) From 1b3904a30755b355f1819ac68fdcc24979f804d2 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 5 Sep 2018 02:13:57 -0500 Subject: [PATCH 68/93] populate common params to commands --- backend/app/cmd/backup.go | 18 +++++---- backend/app/cmd/backup_test.go | 14 ++++--- backend/app/cmd/cmd.go | 6 +++ backend/app/cmd/import.go | 18 +++++---- backend/app/cmd/import_test.go | 23 ++++++++---- backend/app/cmd/restore.go | 23 +++++++----- backend/app/cmd/restore_test.go | 5 ++- backend/app/cmd/server.go | 12 ++++-- backend/app/cmd/server_test.go | 63 +++++++++++++++++++------------- backend/app/main.go | 13 ++++++- backend/app/rest/api/rest.go | 5 --- backend/app/rest/auth/auth.go | 7 ++-- backend/app/store/admin/admin.go | 3 ++ 13 files changed, 131 insertions(+), 79 deletions(-) diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go index 4307e474..96c22387 100644 --- a/backend/app/cmd/backup.go +++ b/backend/app/cmd/backup.go @@ -15,12 +15,16 @@ import ( // BackupCommand set of flags and command for export // ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored. type BackupCommand struct { - ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` - ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"` - Site string `long:"site" env:"SITE" default:"remark" description:"site name"` - SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` - Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` - URL string `long:"url" default:"http://127.0.0.1:8080" description:"base url"` + ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` + ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"` + Site string `long:"site" env:"SITE" default:"remark" description:"site name"` + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + CommonOpts +} + +// SetCommon satisfies main.CommonOptionsCommander interface and sets common options +func (ec *BackupCommand) SetCommon(commonOpts CommonOpts) { + ec.CommonOpts = commonOpts } // Execute runs export with ExportCommand parameters, entry point for "export" command @@ -40,7 +44,7 @@ func (ec *BackupCommand) Execute(args []string) error { client := http.Client{} ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout) defer cancel() - exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret) + exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s&secret=%s", ec.RemarkURL, ec.Site, ec.SharedSecret) req, err := http.NewRequest(http.MethodGet, exportURL, nil) if err != nil { return errors.Wrapf(err, "can't make export request for %s", exportURL) diff --git a/backend/app/cmd/backup_test.go b/backend/app/cmd/backup_test.go index e3d56c11..4eb6eb8f 100644 --- a/backend/app/cmd/backup_test.go +++ b/backend/app/cmd/backup_test.go @@ -22,9 +22,9 @@ func TestBackup_Execute(t *testing.T) { defer ts.Close() cmd := BackupCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", - "--file={{.SITE}}-test.export", "--url=" + ts.URL}) + _, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"}) require.Nil(t, err) err = cmd.Execute(nil) assert.NoError(t, err) @@ -45,9 +45,10 @@ func TestBackup_ExecuteFailedStatus(t *testing.T) { defer ts.Close() cmd := BackupCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) + p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", - "--file={{.SITE}}-test.export", "--url=" + ts.URL}) + _, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"}) require.Nil(t, err) err = cmd.Execute(nil) assert.EqualError(t, err, `error response "400 Bad Request", some error`) @@ -62,9 +63,10 @@ func TestBackup_ExecuteFailedWrite(t *testing.T) { defer ts.Close() cmd := BackupCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) + p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=/tmp", - "--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--url=" + ts.URL}) + _, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file=/tmp/no-such-dir/{{.SITE}}-test.export"}) require.Nil(t, err) err = cmd.Execute(nil) assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`) diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index cb1d2c2f..42b6e2bd 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -19,6 +19,12 @@ import ( // Revision sets from main var Revision = "unknown" +// CommonOpts sets externally from main, shared across all commands +type CommonOpts struct { + RemarkURL string + SharedSecret string +} + // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format type fileParser struct { site string diff --git a/backend/app/cmd/import.go b/backend/app/cmd/import.go index 5e5b3d42..cbb56180 100644 --- a/backend/app/cmd/import.go +++ b/backend/app/cmd/import.go @@ -17,13 +17,16 @@ import ( // ImportCommand set of flags and command for import type ImportCommand struct { - InputFile string `short:"f" long:"file" description:"input file name" required:"true"` - Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` - Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` - SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` + InputFile string `short:"f" long:"file" description:"input file name" required:"true"` + Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` + Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + CommonOpts +} - Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` - URL string `long:"url" default:"http://127.0.0.1:8080" description:"migrator base url"` +// SetCommon satisfies main.CommonOptionsCommander interface and sets common options +func (ic *ImportCommand) SetCommon(commonOpts CommonOpts) { + ic.CommonOpts = commonOpts } // Execute runs import with ImportCommand parameters, entry point for "import" command @@ -39,7 +42,8 @@ func (ic *ImportCommand) Execute(args []string) error { client := http.Client{} ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout) defer cancel() - importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s", ic.URL, ic.Site, ic.Provider, ic.SharedSecret) + importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s", + ic.RemarkURL, ic.Site, ic.Provider, ic.SharedSecret) req, err := http.NewRequest(http.MethodPost, importURL, reader) if err != nil { return errors.Wrapf(err, "can't make import request for %s", importURL) diff --git a/backend/app/cmd/import_test.go b/backend/app/cmd/import_test.go index 30188da5..7e5a9ddb 100644 --- a/backend/app/cmd/import_test.go +++ b/backend/app/cmd/import_test.go @@ -30,15 +30,19 @@ func TestImport_Execute(t *testing.T) { defer ts.Close() cmd := ImportCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) + p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts.URL}) + _, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"}) require.Nil(t, err) err = cmd.Execute(nil) assert.NoError(t, err) cmd = ImportCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) + p = flags.NewParser(&cmd, flags.Default) - _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt.gz", "--url=" + ts.URL}) + _, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz"}) require.Nil(t, err) err = cmd.Execute(nil) assert.NoError(t, err) @@ -54,8 +58,9 @@ func TestImport_ExecuteFailed(t *testing.T) { defer ts.Close() cmd := ImportCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import-no.txt", "--url=" + ts.URL}) + _, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import-no.txt"}) require.Nil(t, err) err = cmd.Execute(nil) t.Log(err) @@ -63,9 +68,9 @@ func TestImport_ExecuteFailed(t *testing.T) { assert.True(t, strings.Contains(err.Error(), "no such file or directory")) cmd = ImportCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: "http://127.0.0.1:12345", SharedSecret: "123456"}) p = flags.NewParser(&cmd, flags.Default) - _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", - "--url=http://127.0.0.1:12345"}) + _, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"}) require.Nil(t, err) err = cmd.Execute(nil) t.Log(err) @@ -79,8 +84,9 @@ func TestImport_ExecuteFailed(t *testing.T) { })) defer ts2.Close() cmd = ImportCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts2.URL, SharedSecret: "123456"}) p = flags.NewParser(&cmd, flags.Default) - _, err = p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", "--url=" + ts2.URL}) + _, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"}) require.Nil(t, err) err = cmd.Execute(nil) t.Log(err) @@ -102,9 +108,10 @@ func TestImport_ExecuteTimeout(t *testing.T) { defer ts.Close() cmd := ImportCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) + p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--file=testdata/import.txt", - "--url=" + ts.URL, "--timeout=300ms"}) + _, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--timeout=300ms"}) require.Nil(t, err) err = cmd.Execute(nil) assert.NotNil(t, err) diff --git a/backend/app/cmd/restore.go b/backend/app/cmd/restore.go index c8e42461..ce9013c5 100644 --- a/backend/app/cmd/restore.go +++ b/backend/app/cmd/restore.go @@ -10,10 +10,14 @@ type RestoreCommand struct { ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"` - Site string `long:"site" env:"SITE" default:"remark" description:"site name"` - SharedSecret string `long:"secret" env:"SECRET" description:"shared secret key" required:"true"` - Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` - URL string `long:"url" default:"http://127.0.0.1:8081" description:"migrator base url"` + Site string `long:"site" env:"SITE" default:"remark" description:"site name"` + Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + CommonOpts +} + +// SetCommon satisfies main.CommonOptionsCommander interface and sets common options +func (rc *RestoreCommand) SetCommon(commonOpts CommonOpts) { + rc.CommonOpts = commonOpts } // Execute runs import with RestoreCommand parameters, entry point for "restore" command @@ -28,12 +32,11 @@ func (rc *RestoreCommand) Execute(args []string) error { return err } importer := ImportCommand{ - InputFile: fname, - Site: rc.Site, - Provider: "native", - SharedSecret: rc.SharedSecret, - Timeout: rc.Timeout, - URL: rc.URL, + InputFile: fname, + Site: rc.Site, + Provider: "native", + Timeout: rc.Timeout, + CommonOpts: rc.CommonOpts, } return importer.Execute(args) } diff --git a/backend/app/cmd/restore_test.go b/backend/app/cmd/restore_test.go index 820968fb..5bd5e8fc 100644 --- a/backend/app/cmd/restore_test.go +++ b/backend/app/cmd/restore_test.go @@ -28,9 +28,10 @@ func TestRestore_Execute(t *testing.T) { defer ts.Close() cmd := RestoreCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"}) + p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--site=remark", "--path=testdata", "--file=import.txt", - "--url=" + ts.URL}) + _, err := p.ParseArgs([]string{"--site=remark", "--path=testdata", "--file=import.txt"}) require.Nil(t, err) err = cmd.Execute(nil) assert.NoError(t, err) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 79c42623..40f7b6d9 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -30,9 +30,6 @@ import ( // ServerCommand with command line flags and env type ServerCommand struct { - RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` - SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` - Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` @@ -64,6 +61,8 @@ type ServerCommand struct { Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"` Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"` } `group:"auth" namespace:"auth" env-namespace:"AUTH"` + + CommonOpts } // AuthGroup defines options group for auth params @@ -131,9 +130,14 @@ type serverApp struct { terminated chan struct{} } +// SetCommon satisfies CommonOptionsCommander interface and sets common options +func (s *ServerCommand) SetCommon(commonOpts CommonOpts) { + s.CommonOpts = commonOpts +} + // Execute is the entry point for "server" command, called by flag parser func (s *ServerCommand) Execute(args []string) error { - log.Print("[INFO] start remark42 server") + log.Printf("[INFO] start server on port %d", s.Port) resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC") ctx, cancel := context.WithCancel(context.Background()) diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 1204f688..93fcb4c7 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -86,10 +86,12 @@ func TestServerApp_WithMongo(t *testing.T) { } opts := ServerCommand{} + opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) + // prepare options p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com", - "--cache.type=mongo", "--store.type=mongo", "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) + _, err := p.ParseArgs([]string{"--dev-passwd=password", "--cache.type=mongo", "--store.type=mongo", + "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) require.Nil(t, err) opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" opts.BackupLocation = "/tmp" @@ -131,11 +133,12 @@ func TestServerApp_WithMongo(t *testing.T) { func TestServerApp_Failed(t *testing.T) { opts := ServerCommand{} + opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) + p := flags.NewParser(&opts, flags.Default) // RO bolt location - _, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", - "--store.bolt.path=/dev/null"}) + _, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"}) assert.Nil(t, err) _, err = opts.newServerApp() assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ @@ -144,8 +147,9 @@ func TestServerApp_Failed(t *testing.T) { // RO backup location opts = ServerCommand{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp", - "--backup=/dev/null/not-writable"}) + opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) + + _, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"}) assert.Nil(t, err) _, err = opts.newServerApp() assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory") @@ -153,14 +157,18 @@ func TestServerApp_Failed(t *testing.T) { // invalid url opts = ServerCommand{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"}) + opts.SetCommon(CommonOpts{RemarkURL: "demo.remark42.com", SharedSecret: "123456"}) + + _, err = p.ParseArgs([]string{"--backup=/tmp", "----store.bolt.path=/tmp"}) assert.Nil(t, err) _, err = opts.newServerApp() assert.EqualError(t, err, "invalid remark42 url demo.remark42.com") t.Log(err) opts = ServerCommand{} - _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"}) + opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) + + _, err = p.ParseArgs([]string{"--backup=/tmp", "--store.type=blah"}) assert.NotNil(t, err, "blah is invalid type") opts.Store.Type = "blah" @@ -191,9 +199,10 @@ func TestServerApp_MainSignal(t *testing.T) { st := time.Now() s := ServerCommand{} + s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) + p := flags.NewParser(&s, flags.Default) - args := []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", - "--port=18100", "--url=https://demo.remark42.com"} + args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", "--port=18100"} _, err := p.ParseArgs(args) require.Nil(t, err) err = s.Execute(args) @@ -202,25 +211,27 @@ func TestServerApp_MainSignal(t *testing.T) { } func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) { - opts := ServerCommand{} - // prepare options - p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"}) - require.Nil(t, err) - opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp" - opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port) - opts.Store.Bolt.Timeout = 10 * time.Second - opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" - opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid" - opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid" - opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid" - opts.BackupLocation = "/tmp" - opts = fn(opts) + cmd := ServerCommand{} + cmd.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) - os.Remove(opts.Store.Bolt.Path + "/remark.db") + // prepare options + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--dev-passwd=password"}) + require.Nil(t, err) + cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp" + cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port) + cmd.Store.Bolt.Timeout = 10 * time.Second + cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid" + cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid" + cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid" + cmd.Auth.Yandex.CSEC, cmd.Auth.Yandex.CID = "csec", "cid" + cmd.BackupLocation = "/tmp" + cmd = fn(cmd) + + os.Remove(cmd.Store.Bolt.Path + "/remark.db") // create app - app, err := opts.newServerApp() + app, err := cmd.newServerApp() require.Nil(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/backend/app/main.go b/backend/app/main.go index 70d3e9aa..0e80e9ff 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -18,11 +18,19 @@ type Opts struct { BackupCmd cmd.BackupCommand `command:"backup"` RestoreCmd cmd.RestoreCommand `command:"restore"` + RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` + SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` + Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` } var revision = "unknown" +type CommonOptionsCommander interface { + SetCommon(commonOpts cmd.CommonOpts) + Execute(args []string) error +} + func main() { fmt.Printf("remark42 %s\n", revision) cmd.Revision = revision @@ -31,7 +39,10 @@ func main() { p := flags.NewParser(&opts, flags.Default) p.CommandHandler = func(command flags.Commander, args []string) error { setupLog(opts.Dbg) - err := command.Execute(args) + commonOpts := cmd.CommonOpts{RemarkURL: opts.RemarkURL, SharedSecret: opts.SharedSecret} + c := command.(CommonOptionsCommander) + c.SetCommon(commonOpts) + err := c.Execute(args) if err != nil { log.Printf("[ERROR] failed with %+v", err) } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index fc3be7ee..ba8738ca 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -70,11 +70,6 @@ type commentsWithInfo struct { func (s *Rest) Run(port int) { log.Printf("[INFO] activate rest server on port %d", port) - // TODO: restore admin info dbg - //if s.DataService != nil && len(s.DataService.AdminStore.Admins()) > 0 { - // log.Printf("[DEBUG] admins %+v", s.DataService.Admins) - //} - router := s.routes() s.lock.Lock() diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index 5e62b1e6..ef642948 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -50,14 +50,15 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { f := func(h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - // if secret key matches and site in request return admin + // if secret key matches for given site (from request) return admin user if a.checkSecretKey(r) { r = rest.SetUserInfo(r, adminUser) h.ServeHTTP(w, r) return } - if a.basicDevUser(w, r) { // use dev user basic auth if enabled + // use dev user basic auth if enabled + if a.basicDevUser(w, r) { r = rest.SetUserInfo(r, devUser) h.ServeHTTP(w, r) return @@ -70,7 +71,7 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { http.Error(w, "Unauthorized", http.StatusUnauthorized) return } - // in anonymous mode just pass it to the next handler + // if !reqAuth just pass it to the next handler, used for information only, like logs h.ServeHTTP(w, r) return } diff --git a/backend/app/store/admin/admin.go b/backend/app/store/admin/admin.go index 3c034972..e75d0ec2 100644 --- a/backend/app/store/admin/admin.go +++ b/backend/app/store/admin/admin.go @@ -1,5 +1,7 @@ package admin +import "log" + // Store defines interface returning admins info for given site type Store interface { Admins(siteID string) (ids []string) @@ -14,6 +16,7 @@ type StaticStore struct { // NewStaticStore makes StaticStore instance with given key func NewStaticStore(admins []string, email string) *StaticStore { + log.Printf("[DEBUG] admin users %+v, email %s", admins, email) return &StaticStore{admins: admins, email: email} } From ba4cc38e406456472ad9d4cad5134d71bf7b77a6 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 5 Sep 2018 02:19:03 -0500 Subject: [PATCH 69/93] lint: missing comment --- backend/app/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/app/main.go b/backend/app/main.go index 0e80e9ff..b0ee2da5 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -26,6 +26,8 @@ type Opts struct { var revision = "unknown" +// CommonOptionsCommander extends flags.Commander with SetCommon +// All commands should implement this interfaces type CommonOptionsCommander interface { SetCommon(commonOpts cmd.CommonOpts) Execute(args []string) error From add715a11174d7e42b202ca323fea4ea35ffd1b3 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 5 Sep 2018 16:42:46 -0500 Subject: [PATCH 70/93] move common options to shared cmd --- backend/app/cmd/backup.go | 5 ----- backend/app/cmd/cmd.go | 13 +++++++++++++ backend/app/cmd/import.go | 5 ----- backend/app/cmd/restore.go | 5 ----- backend/app/cmd/server.go | 5 ----- backend/app/main.go | 9 +-------- 6 files changed, 14 insertions(+), 28 deletions(-) diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go index 96c22387..4f92ad42 100644 --- a/backend/app/cmd/backup.go +++ b/backend/app/cmd/backup.go @@ -22,11 +22,6 @@ type BackupCommand struct { CommonOpts } -// SetCommon satisfies main.CommonOptionsCommander interface and sets common options -func (ec *BackupCommand) SetCommon(commonOpts CommonOpts) { - ec.CommonOpts = commonOpts -} - // Execute runs export with ExportCommand parameters, entry point for "export" command func (ec *BackupCommand) Execute(args []string) error { log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site) diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 42b6e2bd..6997e175 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -19,12 +19,25 @@ import ( // Revision sets from main var Revision = "unknown" +// CommonOptionsCommander extends flags.Commander with SetCommon +// All commands should implement this interfaces +type CommonOptionsCommander interface { + SetCommon(commonOpts CommonOpts) + Execute(args []string) error +} + // CommonOpts sets externally from main, shared across all commands type CommonOpts struct { RemarkURL string SharedSecret string } +// SetCommon satisfies CommonOptionsCommander interface and sets common option fields +func (c *CommonOpts) SetCommon(commonOpts CommonOpts) { + c.RemarkURL = commonOpts.RemarkURL + c.SharedSecret = commonOpts.SharedSecret +} + // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format type fileParser struct { site string diff --git a/backend/app/cmd/import.go b/backend/app/cmd/import.go index cbb56180..d81d4b7c 100644 --- a/backend/app/cmd/import.go +++ b/backend/app/cmd/import.go @@ -24,11 +24,6 @@ type ImportCommand struct { CommonOpts } -// SetCommon satisfies main.CommonOptionsCommander interface and sets common options -func (ic *ImportCommand) SetCommon(commonOpts CommonOpts) { - ic.CommonOpts = commonOpts -} - // Execute runs import with ImportCommand parameters, entry point for "import" command func (ic *ImportCommand) Execute(args []string) error { log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site) diff --git a/backend/app/cmd/restore.go b/backend/app/cmd/restore.go index ce9013c5..fe4589c8 100644 --- a/backend/app/cmd/restore.go +++ b/backend/app/cmd/restore.go @@ -15,11 +15,6 @@ type RestoreCommand struct { CommonOpts } -// SetCommon satisfies main.CommonOptionsCommander interface and sets common options -func (rc *RestoreCommand) SetCommon(commonOpts CommonOpts) { - rc.CommonOpts = commonOpts -} - // Execute runs import with RestoreCommand parameters, entry point for "restore" command // uses ImportCommand with constructed full file name func (rc *RestoreCommand) Execute(args []string) error { diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 40f7b6d9..ebb7167b 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -130,11 +130,6 @@ type serverApp struct { terminated chan struct{} } -// SetCommon satisfies CommonOptionsCommander interface and sets common options -func (s *ServerCommand) SetCommon(commonOpts CommonOpts) { - s.CommonOpts = commonOpts -} - // Execute is the entry point for "server" command, called by flag parser func (s *ServerCommand) Execute(args []string) error { log.Printf("[INFO] start server on port %d", s.Port) diff --git a/backend/app/main.go b/backend/app/main.go index b0ee2da5..00eed26f 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -26,13 +26,6 @@ type Opts struct { var revision = "unknown" -// CommonOptionsCommander extends flags.Commander with SetCommon -// All commands should implement this interfaces -type CommonOptionsCommander interface { - SetCommon(commonOpts cmd.CommonOpts) - Execute(args []string) error -} - func main() { fmt.Printf("remark42 %s\n", revision) cmd.Revision = revision @@ -42,7 +35,7 @@ func main() { p.CommandHandler = func(command flags.Commander, args []string) error { setupLog(opts.Dbg) commonOpts := cmd.CommonOpts{RemarkURL: opts.RemarkURL, SharedSecret: opts.SharedSecret} - c := command.(CommonOptionsCommander) + c := command.(cmd.CommonOptionsCommander) c.SetCommon(commonOpts) err := c.Execute(args) if err != nil { From 061ceb4f81ef39b1b9b0042bcc2b90601157e822 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 5 Sep 2018 20:57:33 -0500 Subject: [PATCH 71/93] lint: unused field in admin --- backend/app/rest/api/admin.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 0dd3d5b3..7cd8ad2e 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -10,7 +10,6 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" - "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/auth" "github.com/umputun/remark/backend/app/rest/cache" @@ -22,7 +21,6 @@ import ( // admin provides router for all requests available for admin users only type admin struct { dataService *service.DataStore - exporter migrator.Exporter cache cache.LoadingCache authenticator auth.Authenticator readOnlyAge int From 2a3b380686a06ac1adadcc8b2d9d8cbbc7950178 Mon Sep 17 00:00:00 2001 From: Alexandr Date: Thu, 6 Sep 2018 09:06:42 +0500 Subject: [PATCH 72/93] UserInfo close by Esc fix --- web/app/components/user-info/user-info.jsx | 14 ++++++++++++++ web/app/embed.js | 3 --- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/web/app/components/user-info/user-info.jsx b/web/app/components/user-info/user-info.jsx index 0ddfbd6e..d31833ec 100644 --- a/web/app/components/user-info/user-info.jsx +++ b/web/app/components/user-info/user-info.jsx @@ -24,6 +24,20 @@ class UserInfo extends Component { .getUserComments({ user: id, limit: 10 }) .then(({ comments = [] }) => this.setState({ comments })) .finally(() => this.setState({ isLoading: false })); + + document.addEventListener('keydown', this.globalOnKeyDown); + } + + componentWillUnmount() { + document.removeEventListener('keydown', this.globalOnKeyDown); + } + + globalOnKeyDown(e) { + // ESCAPE key pressed + if (e.keyCode == 27) { + const data = JSON.stringify({ isUserInfoShown: false }); + window.parent.postMessage(data, '*'); + } } render(props, { comments, isLoading }) { diff --git a/web/app/embed.js b/web/app/embed.js index b801dcc0..9c6b9997 100644 --- a/web/app/embed.js +++ b/web/app/embed.js @@ -151,9 +151,6 @@ function init() { horizontalscrolling="no" />`; this.iframe = this.node.querySelector('iframe'); - this.iframe.onload = () => { - this.iframe.contentDocument.addEventListener('keydown', this.onKeyDown); - }; this.node.appendChild(this.closeEl); document.body.appendChild(this.style); document.body.appendChild(this.back); From d4c36edc55cd1b5183020eeb4b4158f379280318 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 6 Sep 2018 10:35:31 -0500 Subject: [PATCH 73/93] add delete flag to edit request #200 --- README.md | 15 ++++----- backend/app/rest/api/rest_private.go | 2 ++ backend/app/rest/api/rest_private_test.go | 39 +++++++++++++++++++++++ backend/app/store/service/service.go | 6 ++++ backend/app/store/service/service_test.go | 19 +++++++++++ 5 files changed, 73 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a84b3c48..004f68c4 100644 --- a/README.md +++ b/README.md @@ -485,15 +485,14 @@ type Node struct { Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i.e. `-time`. For `tree` mode sort will be applied to top-level comments only and all replies always sorted by time. -* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in 5min since creation +* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation. Body is `EditRequest` json -```json - Content-Type: application/json - - { - "text": "edit comment blah http://radio-t.com 12345", - "summary": "fix blah" - } +```go + type EditRequest struct { + Text string `json:"text"` // updated text + Summary string `json:"summary"` // optional, summary of the edit + Delete bool `json:"delete"` // delete flag + }{} ``` * `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 1a47a895..5c147747 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -83,6 +83,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { edit := struct { Text string Summary string + Delete bool }{} if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil { @@ -112,6 +113,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) { Text: s.CommentFormatter.FormatText(edit.Text), Orig: edit.Text, Summary: edit.Summary, + Delete: edit.Delete, } res, err := s.DataService.EditComment(locator, id, editReq) diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index 7b775ad8..1dd820a9 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -186,6 +186,45 @@ func TestRest_Update(t *testing.T) { assert.Equal(t, c2, c3, "same as response from update") } +func TestRest_UpdateDelete(t *testing.T) { + srv, ts := prep(t) + assert.NotNil(t, srv) + defer cleanup(ts, srv) + + c1 := store.Comment{Text: "test test #1", ParentID: "p1", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} + id := addComment(t, c1, ts) + + client := http.Client{} + req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1", + strings.NewReader(`{"delete": true, "summary":"removed by user"}`)) + assert.Nil(t, err) + req.SetBasicAuth("dev", "password") + b, err := client.Do(req) + assert.Nil(t, err) + body, err := ioutil.ReadAll(b.Body) + assert.Nil(t, err) + assert.Equal(t, 200, b.StatusCode, string(body)) + + // comments returned by update + c2 := store.Comment{} + err = json.Unmarshal(body, &c2) + assert.Nil(t, err) + assert.Equal(t, id, c2.ID) + assert.True(t, c2.Deleted) + + // read updated comment + res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id)) + assert.Equal(t, 200, code) + c3 := store.Comment{} + err = json.Unmarshal([]byte(res), &c3) + assert.Nil(t, err) + assert.Equal(t, "", c3.Text) + assert.Equal(t, "", c3.Orig) + assert.True(t, c3.Deleted) + +} + func TestRest_UpdateNotOwner(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index c9f2d852..8283d5b1 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -124,6 +124,7 @@ type EditRequest struct { Text string Orig string Summary string + Delete bool } // EditComment to edit text and update Edit info @@ -138,6 +139,11 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi return comment, errors.Errorf("too late to edit %s", commentID) } + if req.Delete { // delete request + comment.Deleted = true + return comment, s.Delete(locator, commentID, store.SoftDelete) + } + comment.Text = req.Text comment.Orig = req.Orig comment.Edit = &store.Edit{ diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index 590a65ba..645738ca 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -258,6 +258,25 @@ func TestService_EditComment(t *testing.T) { assert.Nil(t, err, "allow second edit") } +func TestService_DeleteComment(t *testing.T) { + defer os.Remove(testDb) + b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} + + res, err := b.Last("radio-t", 0) + t.Logf("%+v", res[0]) + assert.Nil(t, err) + assert.Equal(t, 2, len(res)) + assert.Nil(t, res[0].Edit) + + _, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Delete: true}) + assert.Nil(t, err) + + c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + assert.Nil(t, err) + assert.True(t, c.Deleted) + t.Logf("%+v", c) +} + func TestService_EditCommentDurationFailed(t *testing.T) { defer os.Remove(testDb) b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, KeyStore: keys.NewStaticStore("secret 123")} From 96e4116d600f6b4dcca85695fe9c5ec98ce83ac5 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 7 Sep 2018 02:43:34 -0500 Subject: [PATCH 74/93] fix typo in comment --- backend/app/cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 6997e175..0eb8e157 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -48,7 +48,7 @@ type fileParser struct { // parse apply template and also concat path and file. In case if file contains path separator path will be ignored func (p *fileParser) parse(now time.Time) (string, error) { - // file/location paramaters my have template masks + // file/location parameters my have template masks fileTemplate := struct { YYYYMMDD string YYYY string From 3ab05490b4fb317f60ba1692e5b8a6305a27d5a5 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 7 Sep 2018 14:48:58 -0500 Subject: [PATCH 75/93] lint: warn on basic auth and non-handled err in gridfs avatar --- backend/app/rest/auth/auth.go | 4 ++-- backend/app/store/avatar/gridfs.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index ef642948..7af61c9e 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -58,7 +58,7 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { } // use dev user basic auth if enabled - if a.basicDevUser(w, r) { + if a.basicDevUser(r) { r = rest.SetUserInfo(r, devUser) h.ServeHTTP(w, r) return @@ -166,7 +166,7 @@ func (a *Authenticator) AdminOnly(next http.Handler) http.Handler { return http.HandlerFunc(fn) } -func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) bool { +func (a *Authenticator) basicDevUser(r *http.Request) bool { if a.DevPasswd == "" { return false diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index f6103f1c..0110b2e1 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -89,7 +89,9 @@ func (gf *GridFS) Remove(avatar string) error { if e != nil { return errors.Wrapf(e, "can't get avatar %s", avatar) } - _ = fh.Close() + if e = fh.Close(); e != nil { + log.Printf("[WARN] can't close avatar %s, %s", avatar, e) + } return dbase.GridFS("fs").Remove(avatar) }) } From 75a1edcff3bc4faecd60c98144f555e2468c62e4 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 9 Sep 2018 16:59:33 -0500 Subject: [PATCH 76/93] feature/adm_mongo (#201) * add admin akd key mongo implementations * wire admin and key stores to mongo * lint: wrong comment on keys store * typos and comments corrections * fix failed test on avatar resizer * get rid of ineffectual assignment in test * add tests for encoding fallback and mongo's keys & admin * empty encode test adjusted --- backend/app/cmd/server.go | 15 +++++++ backend/app/cmd/server_test.go | 3 +- backend/app/migrator/remark_test.go | 2 +- backend/app/rest/api/admin_test.go | 3 +- backend/app/store/admin/admin.go | 51 +++++++++++++++++++++++- backend/app/store/admin/admin_test.go | 43 ++++++++++++++++++++ backend/app/store/avatar/gridfs.go | 2 +- backend/app/store/avatar/localfs.go | 2 +- backend/app/store/avatar/localfs_test.go | 2 +- backend/app/store/formatter.go | 2 +- backend/app/store/formatter_test.go | 4 +- backend/app/store/keys/keys.go | 32 ++++++++++++++- backend/app/store/keys/keys_test.go | 38 ++++++++++++++++++ backend/app/store/user.go | 23 ++++++----- backend/app/store/user_test.go | 24 ++++++++++- 15 files changed, 223 insertions(+), 23 deletions(-) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index ebb7167b..b1bd03f5 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -357,12 +357,20 @@ func (s *ServerCommand) makeKeyStore() (keys.Store, error) { switch s.Key.Type { case "shared": return keys.NewStaticStore(s.SharedSecret), nil + case "mongo": + mgServer, e := s.makeMongo() + if e != nil { + return nil, errors.Wrap(e, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin") + return keys.NewMongoStore(conn), nil default: return nil, errors.Errorf("unsupported key store type %s", s.Key.Type) } } func (s *ServerCommand) makeAdminStore() (admin.Store, error) { + switch s.Admin.Type { case "shared": if s.Admin.Shared.Email == "" { // no admin email, use admin@domain @@ -371,6 +379,13 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) { } } return admin.NewStaticStore(s.Admin.Shared.Admins, s.Admin.Shared.Email), nil + case "mongo": + mgServer, e := s.makeMongo() + if e != nil { + return nil, errors.Wrap(e, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin") + return admin.NewMongoStore(conn), nil default: return nil, errors.Errorf("unsupported admin store type %s", s.Key.Type) } diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 93fcb4c7..5016f7c1 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -91,7 +91,8 @@ func TestServerApp_WithMongo(t *testing.T) { // prepare options p := flags.NewParser(&opts, flags.Default) _, err := p.ParseArgs([]string{"--dev-passwd=password", "--cache.type=mongo", "--store.type=mongo", - "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345"}) + "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", + "--key.type=mongo", "--admin.type=mongo"}) require.Nil(t, err) opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" opts.BackupLocation = "/tmp" diff --git a/backend/app/migrator/remark_test.go b/backend/app/migrator/remark_test.go index 572060d5..7c0621de 100644 --- a/backend/app/migrator/remark_test.go +++ b/backend/app/migrator/remark_test.go @@ -31,7 +31,7 @@ func TestRemark_Export(t *testing.T) { c1, err := buf.ReadString('\n') assert.Nil(t, err) log.Print(c1) - exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" + exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" assert.Equal(t, exp, c1) } diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index 567256d9..9921a99d 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -618,7 +618,8 @@ func TestAdmin_GetUserInfo(t *testing.T) { u := store.User{} err = json.Unmarshal([]byte(body), &u) assert.Nil(t, err) - assert.Equal(t, store.User{Name: "user1 name", ID: "user1", Picture: "", IP: "", Admin: false, Blocked: false, Verified: false}, u) + assert.Equal(t, store.User{Name: "user1 name", ID: "user1", Picture: "", IP: "823688dafca7393d24c871a2da98a84d8732e927", + Admin: false, Blocked: false, Verified: false}, u) _, code = get(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=radio-t&url=https://radio-t.com/blah", ts.URL)) assert.Equal(t, 401, code, "no auth") diff --git a/backend/app/store/admin/admin.go b/backend/app/store/admin/admin.go index e75d0ec2..05068e3e 100644 --- a/backend/app/store/admin/admin.go +++ b/backend/app/store/admin/admin.go @@ -1,6 +1,12 @@ package admin -import "log" +import ( + "log" + + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + "github.com/go-pkgz/mongo" +) // Store defines interface returning admins info for given site type Store interface { @@ -29,3 +35,46 @@ func (s *StaticStore) Admins(string) (ids []string) { func (s *StaticStore) Email(string) (email string) { return s.email } + +// MongoStore implements admin.Store with mongo backend +type MongoStore struct { + connection *mongo.Connection +} + +// NewMongoStore makes admin Store for mongo's connection +func NewMongoStore(conn *mongo.Connection) *MongoStore { + log.Printf("[DEBUG] make mongo admin store with %+v", conn) + return &MongoStore{connection: conn} +} + +// Admins executes find by siteID and returns admins ids +func (m *MongoStore) Admins(siteID string) (ids []string) { + resp := struct { + SiteID string `bson:"site"` + IDs []string `bson:"admin_ids"` + Email string `bson:"admin_email"` + }{} + err := m.connection.WithCollection(func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": siteID}).One(&resp) + }) + if err != nil { + return []string{} + } + return resp.IDs +} + +// Email executes find by siteID and returns admin's email +func (m *MongoStore) Email(siteID string) (email string) { + resp := struct { + SiteID string `bson:"site"` + IDs []string `bson:"admin_ids"` + Email string `bson:"admin_email"` + }{} + err := m.connection.WithCollection(func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": siteID}).One(&resp) + }) + if err != nil { + return "" + } + return resp.Email +} diff --git a/backend/app/store/admin/admin_test.go b/backend/app/store/admin/admin_test.go index acd4df2c..73cf5db3 100644 --- a/backend/app/store/admin/admin_test.go +++ b/backend/app/store/admin/admin_test.go @@ -3,7 +3,10 @@ package admin import ( "testing" + "github.com/globalsign/mgo" + "github.com/go-pkgz/mongo" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStaticStore_Get(t *testing.T) { @@ -15,3 +18,43 @@ func TestStaticStore_Get(t *testing.T) { email := ks.Email("blah") assert.Equal(t, "aa@example.com", email) } + +func TestMongoStore_Get(t *testing.T) { + conn, err := mongo.MakeTestConnection(t) + require.NoError(t, err) + var ms Store = NewMongoStore(conn) + + recs := []struct { + SiteID string `bson:"site"` + IDs []string `bson:"admin_ids"` + Email string `bson:"admin_email"` + }{ + {"site1", []string{"i11", "i12"}, "e1"}, + {"site2", []string{"i21", "i22"}, "e2"}, + } + err = conn.WithCollection(func(coll *mgo.Collection) error { + if e1 := coll.Insert(recs[0]); e1 != nil { + return e1 + } + if e2 := coll.Insert(recs[1]); e2 != nil { + return e2 + } + return nil + }) + require.NoError(t, err) + + admins := ms.Admins("site1") + assert.Equal(t, []string{"i11", "i12"}, admins) + email := ms.Email("site1") + assert.Equal(t, "e1", email) + + admins = ms.Admins("site2") + assert.Equal(t, []string{"i21", "i22"}, admins) + email = ms.Email("site2") + assert.Equal(t, "e2", email) + + admins = ms.Admins("no-site-in-db") + assert.Equal(t, []string{}, admins) + email = ms.Email("no-site-in-db") + assert.Equal(t, "", email) +} diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index 0110b2e1..38a10617 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -40,7 +40,7 @@ func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error // Trying to resize avatar. if reader = resize(reader, gf.resizeLimit); reader == nil { - return errors.New("avatar reader is nil") + return errors.New("avatar resize reader is nil") } _, e = io.Copy(fh, reader) return e diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go index 1c077b0f..306c0221 100644 --- a/backend/app/store/avatar/localfs.go +++ b/backend/app/store/avatar/localfs.go @@ -53,7 +53,7 @@ func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err erro // Trying to resize avatar. if reader = resize(reader, fs.resizeLimit); reader == nil { - return "", errors.New("avatar reader is nil") + return "", errors.New("avatar resize reader is nil") } if _, err = io.Copy(fh, reader); err != nil { diff --git a/backend/app/store/avatar/localfs_test.go b/backend/app/store/avatar/localfs_test.go index 51833f30..2b9d1de7 100644 --- a/backend/app/store/avatar/localfs_test.go +++ b/backend/app/store/avatar/localfs_test.go @@ -19,7 +19,7 @@ func TestAvatarStoreFS_Put(t *testing.T) { avatar, err := p.Put("user1", nil) assert.Equal(t, "", avatar) - assert.EqualError(t, err, "avatar reader is nil") + assert.EqualError(t, err, "avatar resize reader is nil") avatar, err = p.Put("user1", strings.NewReader("some picture bin data")) require.Nil(t, err) diff --git a/backend/app/store/formatter.go b/backend/app/store/formatter.go index d1dec9c5..3543395a 100644 --- a/backend/app/store/formatter.go +++ b/backend/app/store/formatter.go @@ -38,7 +38,7 @@ func (f *CommentFormatter) Format(c Comment) Comment { return c } -// FormatText formatting line +// FormatText converts text with markdown processor, applies external converters and shortens links func (f *CommentFormatter) FormatText(txt string) (res string) { mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode | blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak | diff --git a/backend/app/store/formatter_test.go b/backend/app/store/formatter_test.go index 11f8b609..95959ca3 100644 --- a/backend/app/store/formatter_test.go +++ b/backend/app/store/formatter_test.go @@ -28,12 +28,12 @@ func TestFormatter_FormatText(t *testing.T) { } } -func TestFormatter_FormatTextNoConvertor(t *testing.T) { +func TestFormatter_FormatTextNoConverter(t *testing.T) { f := NewCommentFormatter() assert.Equal(t, "

12345

\n", f.FormatText("12345")) } -func TestFormatter_FormatTextConvertorFunc(t *testing.T) { +func TestFormatter_FormatTextConverterFunc(t *testing.T) { fn := CommentConverterFunc(func(text string) string { return "zz!" + text }) f := NewCommentFormatter(fn) assert.Equal(t, "zz!

12345

\n", f.FormatText("12345")) diff --git a/backend/app/store/keys/keys.go b/backend/app/store/keys/keys.go index 36353a6c..588bf5c6 100644 --- a/backend/app/store/keys/keys.go +++ b/backend/app/store/keys/keys.go @@ -1,6 +1,13 @@ package keys -import "github.com/pkg/errors" +import ( + "log" + + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" +) // Store defines interface returning key for given site // this key used for JWT and HMAC hashes @@ -25,3 +32,26 @@ func (s *StaticStore) Get(siteID string) (key string, err error) { } return s.key, nil } + +// MongoStore implements keys.Store with mongo backend +type MongoStore struct { + connection *mongo.Connection +} + +// NewMongoStore makes keys Store for mongo's connection +func NewMongoStore(conn *mongo.Connection) *MongoStore { + log.Printf("[DEBUG] make mongo keys store with %+v", conn) + return &MongoStore{connection: conn} +} + +// Get executes find by siteID and returns substructure with secret key +func (m *MongoStore) Get(siteID string) (key string, err error) { + resp := struct { + SiteID string `bson:"site"` + SecretKey string `bson:"secret"` + }{} + err = m.connection.WithCollection(func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": siteID}).One(&resp) + }) + return resp.SecretKey, errors.Wrapf(err, "can't get secret for site %s", siteID) +} diff --git a/backend/app/store/keys/keys_test.go b/backend/app/store/keys/keys_test.go index c4b47eb2..0364c200 100644 --- a/backend/app/store/keys/keys_test.go +++ b/backend/app/store/keys/keys_test.go @@ -3,7 +3,10 @@ package keys import ( "testing" + "github.com/globalsign/mgo" + "github.com/go-pkgz/mongo" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStaticStore_Get(t *testing.T) { @@ -18,3 +21,38 @@ func TestStaticStore_Get(t *testing.T) { _, err = ks.Get("any") assert.NotNil(t, err, "invalid (empty key) store") } + +func TestMongoStore_Get(t *testing.T) { + conn, err := mongo.MakeTestConnection(t) + require.NoError(t, err) + var ms Store = NewMongoStore(conn) + + recs := []struct { + SiteID string `bson:"site"` + SecretKey string `bson:"secret"` + }{ + {"site1", "secret1"}, + {"site2", "secret2"}, + } + err = conn.WithCollection(func(coll *mgo.Collection) error { + if e1 := coll.Insert(recs[0]); e1 != nil { + return e1 + } + if e2 := coll.Insert(recs[1]); e2 != nil { + return e2 + } + return nil + }) + require.NoError(t, err) + + r, err := ms.Get("site1") + assert.NoError(t, err) + assert.Equal(t, "secret1", r) + + r, err = ms.Get("site2") + assert.NoError(t, err) + assert.Equal(t, "secret2", r) + + _, err = ms.Get("no-site-in-db") + assert.Error(t, err, "can't get secret for site no-site-in-db") +} diff --git a/backend/app/store/user.go b/backend/app/store/user.go index 626521b4..a0e0cd42 100644 --- a/backend/app/store/user.go +++ b/backend/app/store/user.go @@ -24,6 +24,7 @@ type User struct { } var reValidSha = regexp.MustCompile("^[a-fA-F0-9]{40}$") +var reValidCrc64 = regexp.MustCompile("^[a-fA-F0-9]{16}$") // HashIP replace IP field with hashed hmac func (u *User) HashIP(secret string) { @@ -32,29 +33,29 @@ func (u *User) HashIP(secret string) { // HashValue makes hmac with secret func HashValue(val string, secret string) string { - if val == "" || reValidSha.MatchString(val) { - return val // already hashed or empty - } key := []byte(secret) - h := hmac.New(sha1.New, key) - return hashWithFallback(h, val) + return hashWithFallback(hmac.New(sha1.New, key), val) } // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases // we need hashing for parts of id, in some others hashing for non-User values. func EncodeID(id string) string { - if reValidSha.MatchString(id) { - return id // already hashed or empty - } - h := sha1.New() - return hashWithFallback(h, id) + return hashWithFallback(sha1.New(), id) } -// hashWithFallback tries to has val with hash.Hash and failback to crc if needed +// hashWithFallback tries to has val with hash.Hash and fallback to crc if needed func hashWithFallback(h hash.Hash, val string) string { + + if reValidSha.MatchString(val) { + return val // already hashed or empty + } + if _, err := io.WriteString(h, val); err != nil { // fail back to crc64 log.Printf("[WARN] can't hash id %s, %s", val, err) + if reValidCrc64.MatchString(val) { + return val // already crced + } return fmt.Sprintf("%x", crc64.Checksum([]byte(val), crc64.MakeTable(crc64.ECMA))) } return hex.EncodeToString(h.Sum(nil)) diff --git a/backend/app/store/user_test.go b/backend/app/store/user_test.go index 63f49f60..34a3077a 100644 --- a/backend/app/store/user_test.go +++ b/backend/app/store/user_test.go @@ -1,6 +1,8 @@ package store import ( + "crypto/sha1" + "errors" "testing" "github.com/stretchr/testify/assert" @@ -30,7 +32,7 @@ func TestUser_HashIP(t *testing.T) { {"127.0.0.1", "ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741", "dbc7c999343f003f189f70aaf52cc04443f90790"}, {"8.8.8.8", "8cee77c27e32a2b5aec95c29888ac9946618d9a2", "70a46afce9633f010b06e129b8ad08243a1c4da9"}, {"8cee77c27e32a2b5aec95c29888ac9946618d9a2", "8cee77c27e32a2b5aec95c29888ac9946618d9a2", "8cee77c27e32a2b5aec95c29888ac9946618d9a2"}, - {"", "", ""}, + {"", "fbdb1d1b18aa6c08324b7d64b71fb76370690e1d", "823688dafca7393d24c871a2da98a84d8732e927"}, } for i, tt := range tbl { @@ -43,3 +45,23 @@ func TestUser_HashIP(t *testing.T) { assert.Equal(t, tt.hash2, u.IP, "case #%d", i) } } + +func TestUser_HashFailed(t *testing.T) { + r := hashWithFallback(mockHash{}, "123456789") + assert.Equal(t, "995dc9bbdf1939fa", r) + + r = hashWithFallback(mockHash{}, "995dc9bbdf1939fa") + assert.Equal(t, "995dc9bbdf1939fa", r) + + r = hashWithFallback(sha1.New(), "123456789") + assert.Equal(t, "f7c3bc1d808e04732adf679965ccc34ca7ae3441", r) + +} + +type mockHash struct{} + +func (mock mockHash) Sum(b []byte) []byte { return nil } +func (mock mockHash) Reset() {} +func (mock mockHash) Size() int { return 0 } +func (mock mockHash) BlockSize() int { return 0 } +func (mock mockHash) Write(p []byte) (n int, err error) { return 0, errors.New("error") } From a856fd9c7de70fb1e7bf7a3a2caf2e78aa71f2b1 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 10:44:22 -0500 Subject: [PATCH 77/93] pass revision as common opts instead of package global --- backend/app/cmd/backup_test.go | 2 +- backend/app/cmd/cmd.go | 6 +++--- backend/app/cmd/server.go | 13 +++++++------ backend/app/cmd/server_test.go | 4 ++-- backend/app/main.go | 9 ++++++--- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/backend/app/cmd/backup_test.go b/backend/app/cmd/backup_test.go index 4eb6eb8f..e163d0f4 100644 --- a/backend/app/cmd/backup_test.go +++ b/backend/app/cmd/backup_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - flags "github.com/jessevdk/go-flags" + "github.com/jessevdk/go-flags" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 0eb8e157..664dba0b 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -16,9 +16,6 @@ import ( "github.com/pkg/errors" ) -// Revision sets from main -var Revision = "unknown" - // CommonOptionsCommander extends flags.Commander with SetCommon // All commands should implement this interfaces type CommonOptionsCommander interface { @@ -30,12 +27,15 @@ type CommonOptionsCommander interface { type CommonOpts struct { RemarkURL string SharedSecret string + Revision string } // SetCommon satisfies CommonOptionsCommander interface and sets common option fields +// The method called by main for each command func (c *CommonOpts) SetCommon(commonOpts CommonOpts) { c.RemarkURL = commonOpts.RemarkURL c.SharedSecret = commonOpts.SharedSecret + c.Revision = commonOpts.Revision } // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index b1bd03f5..b1d7ab2f 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -149,7 +149,7 @@ func (s *ServerCommand) Execute(args []string) error { log.Fatalf("[ERROR] failed to setup application, %+v", err) } if err = app.run(ctx); err != nil { - log.Printf("[INFO] remark terminated with error %+v", err) + log.Printf("[WARN] remark terminated with error %+v", err) return err } log.Printf("[INFO] remark terminated") @@ -170,17 +170,18 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { storeEngine, err := s.makeDataStore() if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to make data store engine") } keyStore, err := s.makeKeyStore() if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to make key store") + } adminStore, err := s.makeAdminStore() if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to make admin store") } dataService := &service.DataStore{ @@ -193,7 +194,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { loadingCache, err := s.makeCache() if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to make cache") } // token TTL is 5 minutes, inactivity interval 7+ days by default @@ -225,7 +226,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { commentFormatter := store.NewCommentFormatter(imgProxy) srv := &api.Rest{ - Version: Revision, + Version: s.Revision, DataService: dataService, WebRoot: s.WebRoot, RemarkURL: s.RemarkURL, diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 5016f7c1..c2985795 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -142,7 +142,7 @@ func TestServerApp_Failed(t *testing.T) { _, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"}) assert.Nil(t, err) _, err = opts.newServerApp() - assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ + assert.EqualError(t, err, "failed to make data store engine: can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+ "open /dev/null/remark.db: not a directory") t.Log(err) @@ -174,7 +174,7 @@ func TestServerApp_Failed(t *testing.T) { opts.Store.Type = "blah" _, err = opts.newServerApp() - assert.EqualError(t, err, "unsupported store type blah") + assert.EqualError(t, err, "failed to make data store engine: unsupported store type blah") t.Log(err) } diff --git a/backend/app/main.go b/backend/app/main.go index 00eed26f..36040c70 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -28,15 +28,18 @@ var revision = "unknown" func main() { fmt.Printf("remark42 %s\n", revision) - cmd.Revision = revision var opts Opts p := flags.NewParser(&opts, flags.Default) p.CommandHandler = func(command flags.Commander, args []string) error { setupLog(opts.Dbg) - commonOpts := cmd.CommonOpts{RemarkURL: opts.RemarkURL, SharedSecret: opts.SharedSecret} + // commands implements CommonOptionsCommander to allow passing set of extra options defined for all commands c := command.(cmd.CommonOptionsCommander) - c.SetCommon(commonOpts) + c.SetCommon(cmd.CommonOpts{ + RemarkURL: opts.RemarkURL, + SharedSecret: opts.SharedSecret, + Revision: revision, + }) err := c.Execute(args) if err != nil { log.Printf("[ERROR] failed with %+v", err) From 5b204e509ea3c62ef997e64fa60dcd6a836cf862 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 12:19:52 -0500 Subject: [PATCH 78/93] add List to avatars and allow put without encoding --- backend/app/store/avatar/gridfs.go | 25 +++++++++++++++- backend/app/store/avatar/gridfs_test.go | 33 ++++++++++++++++++++ backend/app/store/avatar/localfs.go | 18 ++++++++++- backend/app/store/avatar/localfs_test.go | 38 ++++++++++++++++++++++++ backend/app/store/avatar/store.go | 20 +++++++++++-- 5 files changed, 129 insertions(+), 5 deletions(-) diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index 38a10617..eb067b1c 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "log" + "time" "github.com/globalsign/mgo" "github.com/go-pkgz/mongo" @@ -26,7 +27,7 @@ type GridFS struct { // Put avatar to gridfs object, try to resize func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) { - id := store.EncodeID(userID) + id := encodeID(userID) err = gf.Connection.WithDB(func(dbase *mgo.Database) error { fh, e := dbase.GridFS("fs").Create(id + imgSfx) if e != nil { @@ -95,3 +96,25 @@ func (gf *GridFS) Remove(avatar string) error { return dbase.GridFS("fs").Remove(avatar) }) } + +func (gf *GridFS) List() (ids []string, err error) { + + type gfsFile struct { + Id interface{} `bson:"_id"` + ChunkSize int `bson:"chunkSize"` + UploadDate time.Time `bson:"uploadDate"` + Length int64 `bson:",minsize"` + MD5 string + Filename string `bson:",omitempty"` + } + + files := []gfsFile{} + err = gf.Connection.WithDB(func(dbase *mgo.Database) error { + return dbase.GridFS("fs").Find(nil).All(&files) + }) + + for _, f := range files { + ids = append(ids, f.Filename) + } + return ids, errors.Wrap(err, "can't list avatars") +} diff --git a/backend/app/store/avatar/gridfs_test.go b/backend/app/store/avatar/gridfs_test.go index 0ba335f0..4ca28741 100644 --- a/backend/app/store/avatar/gridfs_test.go +++ b/backend/app/store/avatar/gridfs_test.go @@ -2,6 +2,7 @@ package avatar import ( "io/ioutil" + "sort" "strings" "testing" @@ -32,6 +33,11 @@ func TestGridFS_PutAndGet(t *testing.T) { assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar)) assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id") + + l, err := p.List() + require.Nil(t, err) + assert.Equal(t, 1, len(l)) + assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0]) } func TestGridFS_Remove(t *testing.T) { @@ -49,6 +55,33 @@ func TestGridFS_Remove(t *testing.T) { assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed") } +func TestGridFS_List(t *testing.T) { + p, skip := prepGFStore(t) + if skip { + return + } + // write some avatars + _, err := p.Put("user1", strings.NewReader("some picture bin data 1")) + require.Nil(t, err) + _, err = p.Put("user2", strings.NewReader("some picture bin data 2")) + require.Nil(t, err) + _, err = p.Put("user3", strings.NewReader("some picture bin data 3")) + require.Nil(t, err) + + l, err := p.List() + assert.NoError(t, err) + assert.Equal(t, 3, len(l), "3 avatars listed") + sort.Strings(l) + assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) + + r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image") + assert.Nil(t, err) + assert.Equal(t, 23, size) + data, err := ioutil.ReadAll(r) + assert.Nil(t, err) + assert.Equal(t, "some picture bin data 3", string(data)) +} + func prepGFStore(t *testing.T) (Store, bool) { conn, err := mongo.MakeTestConnection(t) if err != nil { diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go index 306c0221..6bcae627 100644 --- a/backend/app/store/avatar/localfs.go +++ b/backend/app/store/avatar/localfs.go @@ -7,6 +7,7 @@ import ( "log" "os" "path" + "path/filepath" "strconv" "strings" "sync" @@ -30,8 +31,9 @@ func NewLocalFS(storePath string, resizeLimit int) *LocalFS { } // Put avatar for userID to file and return avatar's file name (base), like 12345678.image +// userID can be avatarID as well, in this case encoding just strip .image prefix func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) { - id := store.EncodeID(userID) + id := encodeID(userID) location := fs.location(id) // location adds partition to path if _, err = os.Stat(location); os.IsNotExist(err) { @@ -95,6 +97,20 @@ func (fs *LocalFS) Remove(avatar string) error { return os.Remove(avFile) } +func (fs *LocalFS) List() (ids []string, err error) { + err = filepath.Walk(fs.storePath, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(info.Name(), imgSfx) { + ids = append(ids, info.Name()) + } + return nil + }) + return ids, errors.Wrap(err, "can't list avatars") +} + // get location (directory) for user id by adding partition to final path in order to keep files // in different subdirectories and avoid too many files in a single place. // the end result is a full path like this - /tmp/avatars.test/92 diff --git a/backend/app/store/avatar/localfs_test.go b/backend/app/store/avatar/localfs_test.go index 2b9d1de7..922c420f 100644 --- a/backend/app/store/avatar/localfs_test.go +++ b/backend/app/store/avatar/localfs_test.go @@ -3,6 +3,7 @@ package avatar import ( "io/ioutil" "os" + "sort" "strings" "testing" "time" @@ -35,6 +36,14 @@ func TestAvatarStoreFS_Put(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(25), fi.Size()) + // with encoded id + avatar, err = p.Put("f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", strings.NewReader("some picture bin data 123")) + require.Nil(t, err) + assert.Equal(t, "f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", avatar) + fi, err = os.Stat("/tmp/avatars.test/56/f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image") + assert.NoError(t, err) + assert.Equal(t, int64(25), fi.Size()) + // with resize file, e := os.Open("testdata/circles.png") require.Nil(t, e) @@ -84,6 +93,7 @@ func TestAvatarStoreFS_Location(t *testing.T) { {"abc", "/tmp/avatars.test/35"}, {"xyz", "/tmp/avatars.test/69"}, {"blah blah", "/tmp/avatars.test/29"}, + {"f1881c06eec96db9901c7bbfe41c42a3f08e9cb8", "/tmp/avatars.test/56"}, } for i, tt := range tbl { @@ -126,6 +136,34 @@ func TestAvatarStoreFS_Remove(t *testing.T) { t.Log(err) } +func TestAvatarStoreFS_List(t *testing.T) { + p := NewLocalFS("/tmp/avatars.test", 300) + err := os.MkdirAll("/tmp/avatars.test", 0700) + require.NoError(t, err) + defer os.RemoveAll("/tmp/avatars.test") + + // write some avatars + _, err = p.Put("user1", strings.NewReader("some picture bin data 1")) + require.Nil(t, err) + _, err = p.Put("user2", strings.NewReader("some picture bin data 2")) + require.Nil(t, err) + _, err = p.Put("user3", strings.NewReader("some picture bin data 3")) + require.Nil(t, err) + + l, err := p.List() + assert.NoError(t, err) + assert.Equal(t, 3, len(l), "3 avatars listed") + sort.Strings(l) + assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) + + r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image") + assert.Nil(t, err) + assert.Equal(t, 23, size) + data, err := ioutil.ReadAll(r) + assert.Nil(t, err) + assert.Equal(t, "some picture bin data 3", string(data)) +} + func BenchmarkAvatarStoreFS_ID(b *testing.B) { p := NewLocalFS("/tmp/avatars.test", 300) os.MkdirAll("/tmp/avatars.test/30", 0700) diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go index 94392405..0a5b5c94 100644 --- a/backend/app/store/avatar/store.go +++ b/backend/app/store/avatar/store.go @@ -6,26 +6,32 @@ package avatar import ( "bytes" "image" - "image/png" - "io" - "log" + "strings" // Initializing packages for supporting GIF and JPEG formats. _ "image/gif" _ "image/jpeg" + "image/png" + "io" + "log" + "regexp" + "github.com/umputun/remark/backend/app/store" "golang.org/x/image/draw" ) // imgSfx for avatars const imgSfx = ".image" +var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`) + // Store defines interface to store and and load avatars type Store interface { Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader ID(avatarID string) (id string) // unique id of stored avatar's data Remove(avatarID string) error // remove avatar data + List() (ids []string, err error) // list all avatar ids } @@ -71,3 +77,11 @@ func resize(reader io.Reader, limit int) io.Reader { } return &out } + +// encodeID converts string to encoded id unless already encoded and valid avatar id (with .image) passed +func encodeID(val string) string { + if reValidAvatarID.MatchString(val) { + return strings.TrimSuffix(val, imgSfx) // already encoded, strip .image + } + return store.EncodeID(val) +} From 9a6ceda00352f142deef2699a86287454dc83816 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 12:38:00 -0500 Subject: [PATCH 79/93] add migrator between avatar stores --- backend/app/store/avatar/gridfs.go | 2 ++ backend/app/store/avatar/localfs.go | 2 ++ backend/app/store/avatar/store.go | 22 +++++++++++++ backend/app/store/avatar/store_test.go | 44 ++++++++++++++++++++++++++ 4 files changed, 70 insertions(+) diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index eb067b1c..5bbe3eb1 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -97,6 +97,8 @@ func (gf *GridFS) Remove(avatar string) error { }) } +// List all avatars (ids) on gfs +// note: id includes .image suffix func (gf *GridFS) List() (ids []string, err error) { type gfsFile struct { diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go index 6bcae627..dd70acad 100644 --- a/backend/app/store/avatar/localfs.go +++ b/backend/app/store/avatar/localfs.go @@ -97,6 +97,8 @@ func (fs *LocalFS) Remove(avatar string) error { return os.Remove(avFile) } +// List all avatars (ids) on local file system +// note: id includes .image suffix func (fs *LocalFS) List() (ids []string, err error) { err = filepath.Walk(fs.storePath, func(path string, info os.FileInfo, err error) error { diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go index 0a5b5c94..add4c83c 100644 --- a/backend/app/store/avatar/store.go +++ b/backend/app/store/avatar/store.go @@ -35,6 +35,28 @@ type Store interface { } +// Migrate avatars between stores +func Migrate(dst Store, src Store) (int, error) { + ids, err := src.List() + if err != nil { + return 0, err + } + for _, id := range ids { + srcReader, _, err := src.Get(id) + if err != nil { + log.Printf("[WARN] can't get reader for avatar %s", id) + continue + } + if _, err = dst.Put(id, srcReader); err != nil { + log.Printf("[WARN] can't put avatar %s", id) + } + if err = srcReader.Close(); err != nil { + log.Printf("[WARN] failed to close avatar %s", id) + } + } + return len(ids), nil +} + // resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side // (width or height) preserving aspect ratio. // Returns original reader if resizing is not needed or failed. diff --git a/backend/app/store/avatar/store_test.go b/backend/app/store/avatar/store_test.go index a0cac0c9..377af11a 100644 --- a/backend/app/store/avatar/store_test.go +++ b/backend/app/store/avatar/store_test.go @@ -5,6 +5,8 @@ import ( "image" "io" "io/ioutil" + "os" + "sort" "strings" "testing" @@ -63,3 +65,45 @@ func TestAvatarStore_resize(t *testing.T) { assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file) } } + +func TestAvatarStore_Migrate(t *testing.T) { + // prep localfs + plocal := NewLocalFS("/tmp/avatars.test", 300) + err := os.MkdirAll("/tmp/avatars.test", 0700) + require.NoError(t, err) + defer os.RemoveAll("/tmp/avatars.test") + + // prep gridfs + pgfs, skip := prepGFStore(t) + if skip { + return + } + + // write to localfs + _, err = plocal.Put("user1", strings.NewReader("some picture bin data 1")) + require.Nil(t, err) + _, err = plocal.Put("user2", strings.NewReader("some picture bin data 2")) + require.Nil(t, err) + _, err = plocal.Put("user3", strings.NewReader("some picture bin data 3")) + require.Nil(t, err) + + // migrate and check reported count + count, err := Migrate(pgfs, plocal) + require.NoError(t, err) + assert.Equal(t, 3, count, "all 3 recs migrated") + + // list avatars + l, err := pgfs.List() + assert.NoError(t, err) + assert.Equal(t, 3, len(l), "3 avatars listed in destination store") + sort.Strings(l) + assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) + + // try to read one of migrated avatars + r, size, err := pgfs.Get("0b7f849446d3383546d15a480966084442cd2193.image") + assert.Nil(t, err) + assert.Equal(t, 23, size) + data, err := ioutil.ReadAll(r) + assert.Nil(t, err) + assert.Equal(t, "some picture bin data 3", string(data)) +} From a231054424503a737e579a8e66f6f0acf84e3932 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 12:41:49 -0500 Subject: [PATCH 80/93] lint: gfs ID mismatch --- backend/app/store/avatar/gridfs.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index 5bbe3eb1..79c925f6 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -102,10 +102,8 @@ func (gf *GridFS) Remove(avatar string) error { func (gf *GridFS) List() (ids []string, err error) { type gfsFile struct { - Id interface{} `bson:"_id"` - ChunkSize int `bson:"chunkSize"` - UploadDate time.Time `bson:"uploadDate"` - Length int64 `bson:",minsize"` + UploadDate time.Time `bson:"uploadDate"` + Length int64 `bson:",minsize"` MD5 string Filename string `bson:",omitempty"` } From 7934d9af736c98e5c74c83b52e9d29bb64caef89 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 20:48:36 -0500 Subject: [PATCH 81/93] add cli for AvatarCmd --- backend/app/cmd/avatar.go | 84 ++++++++++++++++++++++++++++++++++ backend/app/cmd/avatar_test.go | 53 +++++++++++++++++++++ backend/app/cmd/cmd.go | 29 ++++++++++++ backend/app/cmd/server.go | 35 ++------------ backend/app/main.go | 1 + 5 files changed, 170 insertions(+), 32 deletions(-) create mode 100644 backend/app/cmd/avatar.go create mode 100644 backend/app/cmd/avatar_test.go diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go new file mode 100644 index 00000000..c4cd506d --- /dev/null +++ b/backend/app/cmd/avatar.go @@ -0,0 +1,84 @@ +package cmd + +import ( + "log" + "time" + + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store/avatar" +) + +// AvatarCommand set of flags and command for avatar migration +// it converts all avatarts from src.type to dst.type +type AvatarCommand struct { + AvatarSrc AvatarGroup `group:"src" namespace:"src"` + AvatarDst AvatarGroup `group:"dst" namespace:"dst"` + Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` + + migrator AvatarMigrator + CommonOpts +} + +// AvatarMigrator defines interface for migration +type AvatarMigrator interface { + Migrate(avatar.Store, avatar.Store) (int, error) +} + +type avatarMigrator struct{} + +func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) { + return avatar.Migrate(dst, src) +} + +// Execute runs with AvatarCommand parameters, entry point for "avatar" command +func (ac *AvatarCommand) Execute(args []string) error { + log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type) + + src, err := ac.makeAvatarStore(ac.AvatarSrc) + if err != nil { + return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type) + } + + dst, err := ac.makeAvatarStore(ac.AvatarDst) + if err != nil { + return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type) + } + + if ac.migrator == nil { + ac.migrator = avatarMigrator{} + } + + count, err := ac.migrator.Migrate(dst, src) + if err != nil { + return err + } + log.Printf("[INFO] completed, migrated avatars = %d", count) + return nil +} + +func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) { + switch gr.Type { + case "fs": + if err := makeDirs(gr.FS.Path); err != nil { + return nil, err + } + return avatar.NewLocalFS(gr.FS.Path, gr.RszLmt), nil + case "mongo": + mgServer, err := ac.makeMongo() + if err != nil { + return nil, errors.Wrap(err, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "") + return avatar.NewGridFS(conn, gr.RszLmt), nil + } + return nil, errors.Errorf("unsupported avatar store type %s", gr.Type) +} + +func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) { + if ac.Mongo.URL == "" { + return nil, errors.New("no mongo URL provided") + } + return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second) +} diff --git a/backend/app/cmd/avatar_test.go b/backend/app/cmd/avatar_test.go new file mode 100644 index 00000000..5362ae08 --- /dev/null +++ b/backend/app/cmd/avatar_test.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "errors" + "os" + "testing" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/avatar" +) + +func TestAvatar_Execute(t *testing.T) { + + mongoURL := os.Getenv("MONGO_TEST") + if mongoURL == "" { + mongoURL = "mongodb://localhost:27017/test" + } + if mongoURL == "skip" { + t.Skip("skip mongo app test") + } + defer os.RemoveAll("/tmp/ava-test") + + cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}} + cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo", + "--mongo.url=" + mongoURL, "--mongo.db=test_remark"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) + + cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}} + cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo", + "--mongo.url=" + mongoURL, "--mongo.db=test_remark"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.Error(t, err, "failed blah") +} + +type avatarMigratorMock struct { + called int + retError error + retCount int +} + +func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) { + a.called++ + return a.retCount, a.retError +} diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 664dba0b..0b688828 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -98,3 +98,32 @@ func responseError(resp *http.Response) error { } return errors.Errorf("error response %q, %s", resp.Status, body) } + +// mkdir -p for all dirs +func makeDirs(dirs ...string) error { + + // exists returns whether the given file or directory exists or not + exists := func(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return true, err + } + + for _, dir := range dirs { + ex, err := exists(dir) + if err != nil { + return errors.Wrapf(err, "can't check directory status for %s", dir) + } + if !ex { + if e := os.MkdirAll(dir, 0700); e != nil { + return errors.Wrapf(err, "can't make directory %s", dir) + } + } + } + return nil +} diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index b1d7ab2f..acae46b8 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -160,7 +160,7 @@ func (s *ServerCommand) Execute(args []string) error { // doesn't start anything func (s *ServerCommand) newServerApp() (*serverApp, error) { - if err := s.makeDirs(s.BackupLocation); err != nil { + if err := makeDirs(s.BackupLocation); err != nil { return nil, err } @@ -315,7 +315,7 @@ func (a *serverApp) activateBackup(ctx context.Context) { func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { switch s.Store.Type { case "bolt": - if err = s.makeDirs(s.Store.Bolt.Path); err != nil { + if err = makeDirs(s.Store.Bolt.Path); err != nil { return nil, errors.Wrap(err, "failed to create bolt store") } sites := []engine.BoltSite{} @@ -339,7 +339,7 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { switch s.Avatar.Type { case "fs": - if err := s.makeDirs(s.Avatar.FS.Path); err != nil { + if err := makeDirs(s.Avatar.FS.Path); err != nil { return nil, err } return avatar.NewLocalFS(s.Avatar.FS.Path, s.Avatar.RszLmt), nil @@ -451,32 +451,3 @@ func (s *ServerCommand) makeAuthProviders(jwtService *auth.JWT, avatarProxy *pro } return providers } - -// mkdir -p for all dirs -func (s *ServerCommand) makeDirs(dirs ...string) error { - - // exists returns whether the given file or directory exists or not - exists := func(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return true, err - } - - for _, dir := range dirs { - ex, err := exists(dir) - if err != nil { - return errors.Wrapf(err, "can't check directory status for %s", dir) - } - if !ex { - if e := os.MkdirAll(dir, 0700); e != nil { - return errors.Wrapf(err, "can't make directory %s", dir) - } - } - } - return nil -} diff --git a/backend/app/main.go b/backend/app/main.go index 36040c70..33151e55 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -17,6 +17,7 @@ type Opts struct { ImportCmd cmd.ImportCommand `command:"import"` BackupCmd cmd.BackupCommand `command:"backup"` RestoreCmd cmd.RestoreCommand `command:"restore"` + AvatarCmd cmd.AvatarCommand `command:"avatar"` RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"` From 79b1c47005bdf8531ccb6e722e0e23e617188879 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 22:18:23 -0500 Subject: [PATCH 82/93] more dbg info on types --- backend/app/cmd/server.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index acae46b8..1368400b 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -167,6 +167,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") { return nil, errors.Errorf("invalid remark42 url %s", s.RemarkURL) } + log.Printf("[DEBUG] remark42 url=%s", s.RemarkURL) storeEngine, err := s.makeDataStore() if err != nil { @@ -313,6 +314,8 @@ func (a *serverApp) activateBackup(ctx context.Context) { // makeDataStore creates store for all sites func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { + log.Printf("[INFO] make data store, type %s", s.Store.Type) + switch s.Store.Type { case "bolt": if err = makeDirs(s.Store.Bolt.Path); err != nil { @@ -337,6 +340,8 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { } func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { + log.Printf("[INFO] make avatar store, type=%s", s.Avatar.Type) + switch s.Avatar.Type { case "fs": if err := makeDirs(s.Avatar.FS.Path); err != nil { @@ -355,6 +360,8 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { } func (s *ServerCommand) makeKeyStore() (keys.Store, error) { + log.Printf("[INFO] make key store, type=%s", s.Admin.Type) + switch s.Key.Type { case "shared": return keys.NewStaticStore(s.SharedSecret), nil @@ -371,6 +378,7 @@ func (s *ServerCommand) makeKeyStore() (keys.Store, error) { } func (s *ServerCommand) makeAdminStore() (admin.Store, error) { + log.Printf("[INFO] make admin store, type=%s", s.Admin.Type) switch s.Admin.Type { case "shared": @@ -393,6 +401,7 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) { } func (s *ServerCommand) makeCache() (cache.LoadingCache, error) { + log.Printf("[INFO] make cache, type=%s", s.Cache.Type) switch s.Cache.Type { case "mem": return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), @@ -416,12 +425,12 @@ func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) { return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second) } -func (s *ServerCommand) makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore) []auth.Provider { +func (s *ServerCommand) makeAuthProviders(jwt *auth.JWT, ap *proxy.Avatar, ds *service.DataStore) []auth.Provider { makeParams := func(cid, secret string) auth.Params { return auth.Params{ - JwtService: jwtService, - AvatarProxy: avatarProxy, + JwtService: jwt, + AvatarProxy: ap, RemarkURL: s.RemarkURL, Cid: cid, Csecret: secret, From ae8071ac0038c02a5a6ee5af602fff46ff87c556 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 22:18:51 -0500 Subject: [PATCH 83/93] fix web location for binary artifact builds --- Dockerfile.artifacts | 10 ++++++---- compose-dev-backend.yml | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile.artifacts b/Dockerfile.artifacts index 1b739f87..2f890e16 100644 --- a/Dockerfile.artifacts +++ b/Dockerfile.artifacts @@ -30,13 +30,15 @@ WORKDIR /go/src/github.com/umputun/remark/backend ADD backend /go/src/github.com/umputun/remark/backend ADD README.md /go/src/github.com/umputun/remark/ ADD LICENSE /go/src/github.com/umputun/remark/ +COPY --from=build-frontend /srv/web/public/ web -COPY --from=build-frontend /srv/web web - -RUN \ +RUN \ + sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" /go/src/github.com/umputun/remark/backend/web/*.js && \ + sed -i "/REMOVE-START/,/REMOVE-END/d" ./web/iframe.html && \ go get -v github.com/rakyll/statik && \ statik --src=/go/src/github.com/umputun/remark/backend/web --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \ - ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go + ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \ + ls -la /go/src/github.com/umputun/remark/backend/web/ # if DRONE presented use DRONE_* git env to make version RUN \ diff --git a/compose-dev-backend.yml b/compose-dev-backend.yml index 6ad99f6e..9c821c80 100644 --- a/compose-dev-backend.yml +++ b/compose-dev-backend.yml @@ -1,5 +1,5 @@ # compose file for local development -# starts backend on 8080 with basic auth "dev:password" and Dev oauth2 provider on port 8084, UI on https://127.0.0.1:8080/web +# starts backend on 8080 with basic auth "dev:password" and Dev oauth2 provider on port 8084, UI on http://127.0.0.1:8080/web # # mongo-related tests needs mongodb container running - docker run -d -name=mongo mongo:3.6 --smallfiles # start build with backend tests: From 396b77e4ff9648367ffad9fb94f3b75dcea7589a Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 11 Sep 2018 00:36:46 -0500 Subject: [PATCH 84/93] info messages and less verbose auth with dev --- Dockerfile.artifacts | 7 ++++--- backend/app/cmd/server.go | 4 ++-- backend/app/rest/auth/auth.go | 2 -- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile.artifacts b/Dockerfile.artifacts index 2f890e16..ee1254f8 100644 --- a/Dockerfile.artifacts +++ b/Dockerfile.artifacts @@ -33,10 +33,11 @@ ADD LICENSE /go/src/github.com/umputun/remark/ COPY --from=build-frontend /srv/web/public/ web RUN \ - sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" /go/src/github.com/umputun/remark/backend/web/*.js && \ - sed -i "/REMOVE-START/,/REMOVE-END/d" ./web/iframe.html && \ + export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \ + sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \ + sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \ go get -v github.com/rakyll/statik && \ - statik --src=/go/src/github.com/umputun/remark/backend/web --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \ + statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \ ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \ ls -la /go/src/github.com/umputun/remark/backend/web/ diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 1368400b..d2cce995 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -167,7 +167,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") { return nil, errors.Errorf("invalid remark42 url %s", s.RemarkURL) } - log.Printf("[DEBUG] remark42 url=%s", s.RemarkURL) + log.Printf("[INFO] root url=%s", s.RemarkURL) storeEngine, err := s.makeDataStore() if err != nil { @@ -314,7 +314,7 @@ func (a *serverApp) activateBackup(ctx context.Context) { // makeDataStore creates store for all sites func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { - log.Printf("[INFO] make data store, type %s", s.Store.Type) + log.Printf("[INFO] make data store, type=%s", s.Store.Type) switch s.Store.Type { case "bolt": diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index 7af61c9e..cd3d7248 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -172,10 +172,8 @@ func (a *Authenticator) basicDevUser(r *http.Request) bool { return false } - log.Printf("[DEBUG] dev user auth") s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) if len(s) != 2 { - log.Printf("[WARN] dev user auth failed, incorrect auth header %s", r.Header.Get("Authorization")) return false } From 6458f622d23c614235e00f798b1f1f148950afa0 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 11 Sep 2018 11:58:50 -0500 Subject: [PATCH 85/93] refactor to combine admin and key store --- backend/app/cmd/server.go | 41 ++----------- backend/app/cmd/server_test.go | 3 +- backend/app/migrator/disqus_test.go | 4 +- backend/app/migrator/migrator_test.go | 10 ++-- backend/app/migrator/remark_test.go | 9 +-- backend/app/migrator/wordpress_test.go | 4 +- backend/app/rest/api/migrator.go | 8 ++- backend/app/rest/api/migrator_test.go | 14 ++--- backend/app/rest/api/rest_test.go | 8 +-- backend/app/rest/auth/auth.go | 6 +- backend/app/rest/auth/auth_test.go | 12 ++-- backend/app/rest/auth/dev_provider_test.go | 4 +- backend/app/rest/auth/jwt.go | 14 +++-- backend/app/rest/auth/jwt_test.go | 16 ++--- backend/app/rest/auth/provider_test.go | 8 +-- backend/app/store/admin/admin.go | 69 +++++++--------------- backend/app/store/admin/admin_test.go | 28 ++++++--- backend/app/store/admin/mongo.go | 61 +++++++++++++++++++ backend/app/store/keys/keys.go | 57 ------------------ backend/app/store/keys/keys_test.go | 58 ------------------ backend/app/store/service/service.go | 6 +- backend/app/store/service/service_test.go | 26 ++++---- 22 files changed, 181 insertions(+), 285 deletions(-) create mode 100644 backend/app/store/admin/mongo.go delete mode 100644 backend/app/store/keys/keys.go delete mode 100644 backend/app/store/keys/keys_test.go diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index d2cce995..bc1a34ac 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -24,7 +24,6 @@ import ( "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) @@ -34,7 +33,6 @@ type ServerCommand struct { Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` - Key KeyGroup `group:"key" namespace:"key" env-namespace:"KEY"` Admin AdminGroup `group:"admin" namespace:"admin" env-namespace:"ADMIN"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` @@ -105,11 +103,6 @@ type MongoGroup struct { DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` } -// KeyGroup defines options group for key params -type KeyGroup struct { - Type string `long:"type" env:"TYPE" description:"type of key store" choice:"shared" choice:"mongo" default:"shared"` -} - // AdminGroup defines options group for admin params type AdminGroup struct { Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"` @@ -174,12 +167,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { return nil, errors.Wrap(err, "failed to make data store engine") } - keyStore, err := s.makeKeyStore() - if err != nil { - return nil, errors.Wrap(err, "failed to make key store") - - } - adminStore, err := s.makeAdminStore() if err != nil { return nil, errors.Wrap(err, "failed to make admin store") @@ -188,7 +175,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { dataService := &service.DataStore{ Interface: storeEngine, EditDuration: s.EditDuration, - KeyStore: keyStore, AdminStore: adminStore, MaxCommentSize: s.MaxCommentSize, } @@ -199,7 +185,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { } // token TTL is 5 minutes, inactivity interval 7+ days by default - jwtService := auth.NewJWT(keyStore, strings.HasPrefix(s.RemarkURL, "https://"), s.Auth.TTL.JWT, s.Auth.TTL.Cookie) + jwtService := auth.NewJWT(adminStore, strings.HasPrefix(s.RemarkURL, "https://"), s.Auth.TTL.JWT, s.Auth.TTL.Cookie) avatarStore, err := s.makeAvatarStore() if err != nil { @@ -219,7 +205,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { DisqusImporter: &migrator.Disqus{DataStore: dataService}, WordPressImporter: &migrator.WordPress{DataStore: dataService}, NativeExported: &migrator.Remark{DataStore: dataService}, - KeyStore: keyStore, + KeyStore: adminStore, } authProviders := s.makeAuthProviders(jwtService, avatarProxy, dataService) @@ -243,7 +229,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { Providers: authProviders, DevPasswd: s.DevPasswd, PermissionChecker: dataService, - KeyStore: keyStore, }, Cache: loadingCache, } @@ -359,24 +344,6 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type) } -func (s *ServerCommand) makeKeyStore() (keys.Store, error) { - log.Printf("[INFO] make key store, type=%s", s.Admin.Type) - - switch s.Key.Type { - case "shared": - return keys.NewStaticStore(s.SharedSecret), nil - case "mongo": - mgServer, e := s.makeMongo() - if e != nil { - return nil, errors.Wrap(e, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin") - return keys.NewMongoStore(conn), nil - default: - return nil, errors.Errorf("unsupported key store type %s", s.Key.Type) - } -} - func (s *ServerCommand) makeAdminStore() (admin.Store, error) { log.Printf("[INFO] make admin store, type=%s", s.Admin.Type) @@ -387,7 +354,7 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) { s.Admin.Shared.Email = "admin@" + u.Host } } - return admin.NewStaticStore(s.Admin.Shared.Admins, s.Admin.Shared.Email), nil + return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil case "mongo": mgServer, e := s.makeMongo() if e != nil { @@ -396,7 +363,7 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) { conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin") return admin.NewMongoStore(conn), nil default: - return nil, errors.Errorf("unsupported admin store type %s", s.Key.Type) + return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type) } } diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index c2985795..67eddf62 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -91,8 +91,7 @@ func TestServerApp_WithMongo(t *testing.T) { // prepare options p := flags.NewParser(&opts, flags.Default) _, err := p.ParseArgs([]string{"--dev-passwd=password", "--cache.type=mongo", "--store.type=mongo", - "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", - "--key.type=mongo", "--admin.type=mongo"}) + "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"}) require.Nil(t, err) opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" opts.BackupLocation = "/tmp" diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index a8e435e3..cbb021a4 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -9,8 +9,8 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" "github.com/stretchr/testify/assert" @@ -20,7 +20,7 @@ func TestDisqus_Import(t *testing.T) { defer os.Remove("/tmp/remark-test.db") b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} + dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} d := Disqus{DataStore: &dataStore} size, err := d.Import(strings.NewReader(xmlTestDisqus), "test") assert.Nil(t, err) diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 2ec70199..0ebcd09e 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -8,10 +8,10 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store/keys" - "github.com/umputun/remark/backend/app/store/service" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/service" ) func TestMigrator_ImportDisqus(t *testing.T) { @@ -25,7 +25,7 @@ func TestMigrator_ImportDisqus(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} + dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/disqus-test.xml", @@ -51,7 +51,7 @@ func TestMigrator_ImportWordPress(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} + dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/wordpress-test.xml", @@ -80,7 +80,7 @@ func TestMigrator_ImportRemark(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} + dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ DataStore: dataStore, diff --git a/backend/app/migrator/remark_test.go b/backend/app/migrator/remark_test.go index 7c0621de..78a50766 100644 --- a/backend/app/migrator/remark_test.go +++ b/backend/app/migrator/remark_test.go @@ -10,9 +10,10 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) @@ -50,7 +51,7 @@ func TestRemark_Import(t *testing.T) { os.Remove(testDb) b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - r := Remark{DataStore: &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}} + r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} size, err := r.Import(buf, "radio-t") assert.Nil(t, err) assert.Equal(t, 2, size) @@ -78,7 +79,7 @@ func TestRemark_ImportManyWithError(t *testing.T) { os.Remove(testDb) b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - r := Remark{DataStore: &service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")}} + r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} n, err := r.Import(buf, "radio-t") assert.EqualError(t, err, "failed to save 2 comments") assert.Equal(t, 1200, n) @@ -94,7 +95,7 @@ func prep(t *testing.T) *service.DataStore { boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - b := &service.DataStore{Interface: boltStore, KeyStore: keys.NewStaticStore("12345")} + b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")} comment := store.Comment{ ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc", diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go index ae71e4d3..09769b56 100644 --- a/backend/app/migrator/wordpress_test.go +++ b/backend/app/migrator/wordpress_test.go @@ -8,9 +8,9 @@ import ( "github.com/coreos/bbolt" "github.com/stretchr/testify/assert" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) @@ -21,7 +21,7 @@ func TestWordPress_Import(t *testing.T) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) assert.Nil(t, err, "create store") - dataStore := service.DataStore{Interface: b, KeyStore: keys.NewStaticStore("12345")} + dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} wp := WordPress{DataStore: &dataStore} size, err := wp.Import(strings.NewReader(xmlTestWP), siteID) assert.Nil(t, err) diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 01534e4d..b781cc20 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -14,7 +14,6 @@ import ( "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/cache" - "github.com/umputun/remark/backend/app/store/keys" ) // Migrator rest with import and export controllers @@ -24,7 +23,12 @@ type Migrator struct { DisqusImporter migrator.Importer WordPressImporter migrator.Importer NativeExported migrator.Exporter - KeyStore keys.Store + KeyStore KeyStore +} + +// KeyStore defines sub-interface for consumers needed just a key +type KeyStore interface { + Key(siteID string) (key string, err error) } func (m *Migrator) withRoutes(router chi.Router) chi.Router { diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index eb0c2b52..6771d37a 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -22,7 +22,6 @@ import ( "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) @@ -153,29 +152,28 @@ func TestMigrator_Export(t *testing.T) { func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) - ks := keys.NewStaticStore("123456") - dataStore := &service.DataStore{Interface: b, KeyStore: ks} + adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com") + dataStore := &service.DataStore{Interface: b, AdminStore: adminStore} svc = &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, NativeImporter: &migrator.Remark{DataStore: dataStore}, NativeExported: &migrator.Remark{DataStore: dataStore}, Cache: &cache.Nop{}, - KeyStore: ks, + KeyStore: adminStore, } a := auth.Authenticator{ DevPasswd: "password", Providers: nil, - AdminStore: adminstore.NewStaticStore([]string{"a1", "a2"}, "admin@remark-42.com"), - JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), - KeyStore: ks, + AdminStore: adminStore, + JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour), } routes := svc.withRoutes(chi.NewRouter().With(a.Auth(true)).With(a.AdminOnly)) ts = httptest.NewServer(routes) return svc, dataStore, ts } -func cleanupImportSrv(m *Migrator, ts *httptest.Server) { +func cleanupImportSrv(_ *Migrator, ts *httptest.Server) { ts.Close() os.Remove(testDb) } diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index df27dacf..666caed5 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -23,7 +23,6 @@ import ( adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" "github.com/umputun/remark/backend/app/store/service" ) @@ -92,13 +91,12 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) - adminStore := adminstore.NewStaticStore([]string{"a1", "a2"}, "admin@remark-42.com") + adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com") dataStore := &service.DataStore{ Interface: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, - KeyStore: keys.NewStaticStore("123456"), AdminStore: adminStore, } srv = &Rest{ @@ -107,7 +105,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { DevPasswd: "password", Providers: nil, AdminStore: adminStore, - JWTService: auth.NewJWT(keys.NewStaticStore("123456"), false, time.Minute, time.Hour), + JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour), }, Cache: &cache.Nop{}, WebRoot: "/tmp", @@ -122,7 +120,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { NativeImporter: &migrator.Remark{DataStore: dataStore}, NativeExported: &migrator.Remark{DataStore: dataStore}, Cache: &cache.Nop{}, - KeyStore: keys.NewStaticStore("123456"), + KeyStore: adminStore, }, } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10 diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index cd3d7248..758924ab 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -10,7 +10,6 @@ import ( "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/keys" ) // Authenticator is top level auth object providing middlewares @@ -18,7 +17,6 @@ type Authenticator struct { JWTService *JWT Providers []Provider AdminStore admin.Store - KeyStore keys.Store DevPasswd string PermissionChecker PermissionChecker } @@ -114,14 +112,14 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { } func (a *Authenticator) checkSecretKey(r *http.Request) bool { - if a.KeyStore == nil { + if a.AdminStore == nil { return false } siteID := r.URL.Query().Get("site") secret := r.URL.Query().Get("secret") - skey, err := a.KeyStore.Get(siteID) + skey, err := a.AdminStore.Key(siteID) if err != nil { return false } diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index 4efc80f9..2216f1bd 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -11,7 +11,7 @@ import ( "github.com/go-chi/chi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store/keys" + "github.com/umputun/remark/backend/app/store/admin" ) var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOnRydWV9LCJzdGF0ZSI6IjEyMzQ1NiIsImZyb20iOiJmcm9tIn0.6P_OwGf8CUJRtvNSlW20GmaMb5pFvCNemP94fHCqb5Q" @@ -19,7 +19,7 @@ var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxO var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZmxhZ3MiOnsiZGVsZXRlbWUiOnRydWV9fQ.SLh1QpFytWZqcT99VgcdAOtgFKhvpKCcZwqWTvAd63g" func TestAuthJWTCookie(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour), + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour), PermissionChecker: &mockUserPermissions{}} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { @@ -57,7 +57,7 @@ func TestAuthJWTCookie(t *testing.T) { } func TestAuthJWTHeader(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)} + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) @@ -84,7 +84,7 @@ func TestAuthJWTHeader(t *testing.T) { } func TestAuthJWtBlocked(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)} + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) @@ -104,7 +104,7 @@ func TestAuthJWtBlocked(t *testing.T) { } func TestAuthJWtFlags(t *testing.T) { - a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, time.Hour)} + a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)} router := chi.NewRouter() router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) @@ -213,7 +213,7 @@ func TestAdminRequired(t *testing.T) { } func TestAuthWithSecret(t *testing.T) { - a := Authenticator{DevPasswd: "123456", KeyStore: keys.NewStaticStore("secretkey")} + a := Authenticator{DevPasswd: "123456", AdminStore: admin.NewStaticKeyStore("secretkey")} router := chi.NewRouter() router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) diff --git a/backend/app/rest/auth/dev_provider_test.go b/backend/app/rest/auth/dev_provider_test.go index 8f768baa..6a732bfe 100644 --- a/backend/app/rest/auth/dev_provider_test.go +++ b/backend/app/rest/auth/dev_provider_test.go @@ -13,12 +13,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/keys" + "github.com/umputun/remark/backend/app/store/admin" ) func TestDevProvider(t *testing.T) { params := Params{RemarkURL: "http://127.0.0.1:8080", Cid: "cid", Csecret: "csecret", - JwtService: NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31), + JwtService: NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31), PermissionChecker: &mockUserPermissions{admin: "dev_user"}, } srv := DevAuthServer{Provider: NewDev(params), nonInteractive: true, username: "dev_user"} diff --git a/backend/app/rest/auth/jwt.go b/backend/app/rest/auth/jwt.go index d6fd3c06..a95f2934 100644 --- a/backend/app/rest/auth/jwt.go +++ b/backend/app/rest/auth/jwt.go @@ -8,13 +8,12 @@ import ( "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/keys" ) // JWT wraps jwt operations // supports both header and cookie jwt type JWT struct { - keyStore keys.Store + keyStore KeyStore secureCookies bool tokenDuration time.Duration cookieDuration time.Duration @@ -43,8 +42,13 @@ const jwtHeaderKey = "X-JWT" const xsrfCookieName = "XSRF-TOKEN" const xsrfHeaderKey = "X-XSRF-TOKEN" +// KeyStore defines sub-interface for consumers needed just a key +type KeyStore interface { + Key(siteID string) (key string, err error) +} + // NewJWT makes JWT service -func NewJWT(keyStore keys.Store, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT { +func NewJWT(keyStore KeyStore, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT { res := JWT{ keyStore: keyStore, secureCookies: secureCookies, @@ -58,7 +62,7 @@ func NewJWT(keyStore keys.Store, secureCookies bool, tokenDuration time.Duration func (j *JWT) Token(claims *CustomClaims) (string, error) { token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - secret, err := j.keyStore.Get(claims.SiteID) + secret, err := j.keyStore.Key(claims.SiteID) if err != nil { return "", errors.Wrap(err, "can't get secret") } @@ -96,7 +100,7 @@ func (j *JWT) Parse(tokenString string) (*CustomClaims, error) { return nil, errors.Wrap(err, "failed to get siteID from jwt token") } - secret, err := j.keyStore.Get(siteID) + secret, err := j.keyStore.Key(siteID) if err != nil { return nil, errors.Wrap(err, "can't get secret") } diff --git a/backend/app/rest/auth/jwt_test.go b/backend/app/rest/auth/jwt_test.go index f1ce7efa..f1e654d8 100644 --- a/backend/app/rest/auth/jwt_test.go +++ b/backend/app/rest/auth/jwt_test.go @@ -10,7 +10,7 @@ import ( "github.com/dgrijalva/jwt-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store/keys" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store" ) @@ -30,7 +30,7 @@ var testJwtBadSign = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4M var days31 = time.Hour * 24 * 31 func TestJWT_Token(t *testing.T) { - j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) + j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -53,7 +53,7 @@ func TestJWT_Token(t *testing.T) { } func TestJWT_Parse(t *testing.T) { - j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) + j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31) claims, err := j.Parse(testJwtValid) assert.NoError(t, err) assert.False(t, j.IsExpired(claims)) @@ -71,7 +71,7 @@ func TestJWT_Parse(t *testing.T) { } func TestJWT_Set(t *testing.T) { - j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) + j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -116,7 +116,7 @@ func TestJWT_Set(t *testing.T) { } func TestJWT_GetFromHeader(t *testing.T) { - j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) + j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31) req := httptest.NewRequest("GET", "/", nil) req.Header.Add(jwtHeaderKey, testJwtValid) @@ -141,7 +141,7 @@ func TestJWT_GetFromHeader(t *testing.T) { } func TestJWT_SetAndGetWithCookies(t *testing.T) { - j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) + j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -183,7 +183,7 @@ func TestJWT_SetAndGetWithCookies(t *testing.T) { } func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) { - j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) + j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", @@ -220,7 +220,7 @@ func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) { } func TestJWT_SetAndGetWithCookiesExpired(t *testing.T) { - j := NewJWT(keys.NewStaticStore("xyz 12345"), false, time.Hour, days31) + j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31) claims := &CustomClaims{ State: "123456", diff --git a/backend/app/rest/auth/provider_test.go b/backend/app/rest/auth/provider_test.go index be78d789..c2b41616 100644 --- a/backend/app/rest/auth/provider_test.go +++ b/backend/app/rest/auth/provider_test.go @@ -13,10 +13,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store/keys" "golang.org/x/oauth2" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" ) func TestLogin(t *testing.T) { @@ -54,7 +54,7 @@ func TestLogin(t *testing.T) { Admin: false, Blocked: true, IP: ""}, u) token := resp.Cookies()[0].Value - jwtSvc := NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31) + jwtSvc := NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31) claims, err := jwtSvc.Parse(token) require.NoError(t, err) @@ -103,7 +103,7 @@ func TestLoginSessionOnly(t *testing.T) { req.AddCookie(resp.Cookies()[1]) req.Header.Add("X-XSRF-TOKEN", resp.Cookies()[1].Value) - jwtService := NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour) + jwtService := NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour) res, err := jwtService.Get(req) require.Nil(t, err) assert.Equal(t, true, res.SessionOnly) @@ -169,7 +169,7 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se } params := Params{RemarkURL: "url", Cid: "cid", Csecret: "csecret", - JwtService: NewJWT(keys.NewStaticStore("12345"), false, time.Hour, time.Hour*24*31), + JwtService: NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31), // AvatarProxy: &proxy.Avatar{Store: &mockAvatarStore, RoutePath: "/v1/avatar"}, PermissionChecker: &mockUserPermissions{admin: "mock_myuser2", verified: "mock_myuser2", blocked: "mock_myuser1"}, } diff --git a/backend/app/store/admin/admin.go b/backend/app/store/admin/admin.go index 05068e3e..2e2815b4 100644 --- a/backend/app/store/admin/admin.go +++ b/backend/app/store/admin/admin.go @@ -1,15 +1,15 @@ +// Package admin defines and implements store for admin-level data like secret key, list of admins and so on + package admin import ( + "errors" "log" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/go-pkgz/mongo" ) // Store defines interface returning admins info for given site type Store interface { + Key(siteID string) (key string, err error) Admins(siteID string) (ids []string) Email(siteID string) (email string) } @@ -18,12 +18,26 @@ type Store interface { type StaticStore struct { admins []string email string + key string +} + +// Key returns static key for all sites, allows empty site +func (s *StaticStore) Key(siteID string) (key string, err error) { + if s.key == "" { + return "", errors.New("empty key for static key store") + } + return s.key, nil } // NewStaticStore makes StaticStore instance with given key -func NewStaticStore(admins []string, email string) *StaticStore { +func NewStaticStore(key string, admins []string, email string) *StaticStore { log.Printf("[DEBUG] admin users %+v, email %s", admins, email) - return &StaticStore{admins: admins, email: email} + return &StaticStore{key: key, admins: admins, email: email} +} + +// NewStaticKeyStore is a shortcut for making StaticStore for key consumers only +func NewStaticKeyStore(key string) *StaticStore { + return &StaticStore{key: key, admins: []string{}, email: ""} } // Admins returns static list of admin's ids, the same for all sites @@ -35,46 +49,3 @@ func (s *StaticStore) Admins(string) (ids []string) { func (s *StaticStore) Email(string) (email string) { return s.email } - -// MongoStore implements admin.Store with mongo backend -type MongoStore struct { - connection *mongo.Connection -} - -// NewMongoStore makes admin Store for mongo's connection -func NewMongoStore(conn *mongo.Connection) *MongoStore { - log.Printf("[DEBUG] make mongo admin store with %+v", conn) - return &MongoStore{connection: conn} -} - -// Admins executes find by siteID and returns admins ids -func (m *MongoStore) Admins(siteID string) (ids []string) { - resp := struct { - SiteID string `bson:"site"` - IDs []string `bson:"admin_ids"` - Email string `bson:"admin_email"` - }{} - err := m.connection.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID}).One(&resp) - }) - if err != nil { - return []string{} - } - return resp.IDs -} - -// Email executes find by siteID and returns admin's email -func (m *MongoStore) Email(siteID string) (email string) { - resp := struct { - SiteID string `bson:"site"` - IDs []string `bson:"admin_ids"` - Email string `bson:"admin_email"` - }{} - err := m.connection.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID}).One(&resp) - }) - if err != nil { - return "" - } - return resp.Email -} diff --git a/backend/app/store/admin/admin_test.go b/backend/app/store/admin/admin_test.go index 73cf5db3..ebbbec1e 100644 --- a/backend/app/store/admin/admin_test.go +++ b/backend/app/store/admin/admin_test.go @@ -10,13 +10,21 @@ import ( ) func TestStaticStore_Get(t *testing.T) { - var ks Store = NewStaticStore([]string{"123", "xyz"}, "aa@example.com") + var ks Store = NewStaticStore("key123", []string{"123", "xyz"}, "aa@example.com") + + k, err := ks.Key("any") + assert.NoError(t, err, "valid store") + assert.Equal(t, "key123", k, "valid site") a := ks.Admins("any") assert.Equal(t, []string{"123", "xyz"}, a) email := ks.Email("blah") assert.Equal(t, "aa@example.com", email) + + ks = NewStaticStore("", []string{"123", "xyz"}, "aa@example.com") + _, err = ks.Key("any") + assert.NotNil(t, err, "invalid (empty key) store") } func TestMongoStore_Get(t *testing.T) { @@ -24,13 +32,9 @@ func TestMongoStore_Get(t *testing.T) { require.NoError(t, err) var ms Store = NewMongoStore(conn) - recs := []struct { - SiteID string `bson:"site"` - IDs []string `bson:"admin_ids"` - Email string `bson:"admin_email"` - }{ - {"site1", []string{"i11", "i12"}, "e1"}, - {"site2", []string{"i21", "i22"}, "e2"}, + recs := []mongoRec{ + {"site1", "secret1", []string{"i11", "i12"}, "e1"}, + {"site2", "secret2", []string{"i21", "i22"}, "e2"}, } err = conn.WithCollection(func(coll *mgo.Collection) error { if e1 := coll.Insert(recs[0]); e1 != nil { @@ -47,14 +51,22 @@ func TestMongoStore_Get(t *testing.T) { assert.Equal(t, []string{"i11", "i12"}, admins) email := ms.Email("site1") assert.Equal(t, "e1", email) + key, err := ms.Key("site1") + assert.NoError(t, err) + assert.Equal(t, "secret1", key) admins = ms.Admins("site2") assert.Equal(t, []string{"i21", "i22"}, admins) email = ms.Email("site2") assert.Equal(t, "e2", email) + key, err = ms.Key("site2") + assert.NoError(t, err) + assert.Equal(t, "secret2", key) admins = ms.Admins("no-site-in-db") assert.Equal(t, []string{}, admins) email = ms.Email("no-site-in-db") assert.Equal(t, "", email) + _, err = ms.Key("no-site-in-db") + assert.Error(t, err, "can't get secret for site no-site-in-db") } diff --git a/backend/app/store/admin/mongo.go b/backend/app/store/admin/mongo.go new file mode 100644 index 00000000..ddd98ac8 --- /dev/null +++ b/backend/app/store/admin/mongo.go @@ -0,0 +1,61 @@ +package admin + +import ( + "log" + + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" +) + +// MongoStore implements admin.Store with mongo backend +type MongoStore struct { + connection *mongo.Connection +} + +type mongoRec struct { + SiteID string `bson:"site"` + SecretKey string `bson:"secret"` + IDs []string `bson:"admin_ids"` + Email string `bson:"admin_email"` +} + +// NewMongoStore makes admin Store for mongo's connection +func NewMongoStore(conn *mongo.Connection) *MongoStore { + log.Printf("[DEBUG] make mongo admin store with %+v", conn) + return &MongoStore{connection: conn} +} + +// Key executes find by siteID and returns substructure with secret key +func (m *MongoStore) Key(siteID string) (key string, err error) { + resp := mongoRec{} + err = m.connection.WithCollection(func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": siteID}).One(&resp) + }) + return resp.SecretKey, errors.Wrapf(err, "can't get secret for site %s", siteID) +} + +// Admins executes find by siteID and returns admins ids +func (m *MongoStore) Admins(siteID string) (ids []string) { + resp := mongoRec{} + err := m.connection.WithCollection(func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": siteID}).One(&resp) + }) + if err != nil { + return []string{} + } + return resp.IDs +} + +// Email executes find by siteID and returns admin's email +func (m *MongoStore) Email(siteID string) (email string) { + resp := mongoRec{} + err := m.connection.WithCollection(func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": siteID}).One(&resp) + }) + if err != nil { + return "" + } + return resp.Email +} diff --git a/backend/app/store/keys/keys.go b/backend/app/store/keys/keys.go deleted file mode 100644 index 588bf5c6..00000000 --- a/backend/app/store/keys/keys.go +++ /dev/null @@ -1,57 +0,0 @@ -package keys - -import ( - "log" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/go-pkgz/mongo" - "github.com/pkg/errors" -) - -// Store defines interface returning key for given site -// this key used for JWT and HMAC hashes -type Store interface { - Get(siteID string) (key string, err error) -} - -// StaticStore implements keys.Store with a single, predefined key -type StaticStore struct { - key string -} - -// NewStaticStore makes StaticStore instance with given key -func NewStaticStore(key string) *StaticStore { - return &StaticStore{key: key} -} - -// Get returns static key for all sites, allows empty site -func (s *StaticStore) Get(siteID string) (key string, err error) { - if s.key == "" { - return "", errors.New("empty key for static key store") - } - return s.key, nil -} - -// MongoStore implements keys.Store with mongo backend -type MongoStore struct { - connection *mongo.Connection -} - -// NewMongoStore makes keys Store for mongo's connection -func NewMongoStore(conn *mongo.Connection) *MongoStore { - log.Printf("[DEBUG] make mongo keys store with %+v", conn) - return &MongoStore{connection: conn} -} - -// Get executes find by siteID and returns substructure with secret key -func (m *MongoStore) Get(siteID string) (key string, err error) { - resp := struct { - SiteID string `bson:"site"` - SecretKey string `bson:"secret"` - }{} - err = m.connection.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID}).One(&resp) - }) - return resp.SecretKey, errors.Wrapf(err, "can't get secret for site %s", siteID) -} diff --git a/backend/app/store/keys/keys_test.go b/backend/app/store/keys/keys_test.go deleted file mode 100644 index 0364c200..00000000 --- a/backend/app/store/keys/keys_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package keys - -import ( - "testing" - - "github.com/globalsign/mgo" - "github.com/go-pkgz/mongo" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestStaticStore_Get(t *testing.T) { - var ks Store = NewStaticStore("key123") - - k, err := ks.Get("any") - assert.NoError(t, err, "valid store") - assert.Equal(t, "key123", k, "valid site") - - ks = NewStaticStore("") - - _, err = ks.Get("any") - assert.NotNil(t, err, "invalid (empty key) store") -} - -func TestMongoStore_Get(t *testing.T) { - conn, err := mongo.MakeTestConnection(t) - require.NoError(t, err) - var ms Store = NewMongoStore(conn) - - recs := []struct { - SiteID string `bson:"site"` - SecretKey string `bson:"secret"` - }{ - {"site1", "secret1"}, - {"site2", "secret2"}, - } - err = conn.WithCollection(func(coll *mgo.Collection) error { - if e1 := coll.Insert(recs[0]); e1 != nil { - return e1 - } - if e2 := coll.Insert(recs[1]); e2 != nil { - return e2 - } - return nil - }) - require.NoError(t, err) - - r, err := ms.Get("site1") - assert.NoError(t, err) - assert.Equal(t, "secret1", r) - - r, err = ms.Get("site2") - assert.NoError(t, err) - assert.Equal(t, "secret2", r) - - _, err = ms.Get("no-site-in-db") - assert.Error(t, err, "can't get secret for site no-site-in-db") -} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 8283d5b1..2d0ae63a 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -6,18 +6,16 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" - "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/keys" ) // DataStore wraps store.Interface with additional methods type DataStore struct { engine.Interface EditDuration time.Duration - KeyStore keys.Store AdminStore admin.Store MaxCommentSize int @@ -56,7 +54,7 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err } comment.Sanitize() // clear potentially dangerous js from all parts of comment - secret, err := s.KeyStore.Get(comment.Locator.SiteID) + secret, err := s.AdminStore.Key(comment.Locator.SiteID) if err != nil { return store.Comment{}, errors.Wrapf(err, "can't get secret for site %s", comment.Locator.SiteID) } diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index 645738ca..55442340 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -13,7 +13,7 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store/keys" + "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/engine" @@ -23,8 +23,8 @@ var testDb = "/tmp/test-remark.db" func TestService_CreateFromEmpty(t *testing.T) { defer os.Remove(testDb) - ks := keys.NewStaticStore("secret 123") - b := DataStore{Interface: prepStoreEngine(t), KeyStore: ks} + ks := admin.NewStaticKeyStore("secret 123") + b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks} comment := store.Comment{ Text: "text", User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"}, @@ -47,8 +47,8 @@ func TestService_CreateFromEmpty(t *testing.T) { func TestService_CreateFromPartial(t *testing.T) { defer os.Remove(testDb) - ks := keys.NewStaticStore("secret 123") - b := DataStore{Interface: prepStoreEngine(t), KeyStore: ks} + ks := admin.NewStaticKeyStore("secret 123") + b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks} comment := store.Comment{ Text: "text", Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC), @@ -73,7 +73,7 @@ func TestService_CreateFromPartial(t *testing.T) { func TestService_Vote(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} comment := store.Comment{ Text: "text", @@ -118,7 +118,7 @@ func TestService_Vote(t *testing.T) { func TestService_VoteAggressive(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} comment := store.Comment{ Text: "text", @@ -178,7 +178,7 @@ func TestService_VoteAggressive(t *testing.T) { func TestService_VoteConcurrent(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} comment := store.Comment{ Text: "text", @@ -209,7 +209,7 @@ func TestService_VoteConcurrent(t *testing.T) { func TestService_Pin(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -233,7 +233,7 @@ func TestService_Pin(t *testing.T) { func TestService_EditComment(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -260,7 +260,7 @@ func TestService_EditComment(t *testing.T) { func TestService_DeleteComment(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -279,7 +279,7 @@ func TestService_DeleteComment(t *testing.T) { func TestService_EditCommentDurationFailed(t *testing.T) { defer os.Remove(testDb) - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")} res, err := b.Last("radio-t", 0) t.Logf("%+v", res[0]) @@ -296,7 +296,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) { func TestService_ValidateComment(t *testing.T) { - b := DataStore{MaxCommentSize: 2000, KeyStore: keys.NewStaticStore("secret 123")} + b := DataStore{MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")} longText := fmt.Sprintf("%4000s", "X") tbl := []struct { From a8b92a60396bff8d2d11d466c6613165da3f614b Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 11 Sep 2018 12:08:10 -0500 Subject: [PATCH 86/93] lint: detached package comment --- backend/app/store/admin/admin.go | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/store/admin/admin.go b/backend/app/store/admin/admin.go index 2e2815b4..10799ccd 100644 --- a/backend/app/store/admin/admin.go +++ b/backend/app/store/admin/admin.go @@ -1,5 +1,4 @@ // Package admin defines and implements store for admin-level data like secret key, list of admins and so on - package admin import ( From bb0096dd3d6cca78af14533878f56f084e72ca00 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 11 Sep 2018 14:22:02 -0500 Subject: [PATCH 87/93] add test for empty user in a token --- backend/app/rest/auth/auth_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index 2216f1bd..c69df4d9 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -18,6 +18,8 @@ var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxO var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZmxhZ3MiOnsiZGVsZXRlbWUiOnRydWV9fQ.SLh1QpFytWZqcT99VgcdAOtgFKhvpKCcZwqWTvAd63g" +var testJwtNoUser = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyfQ.sBpblkbBRzZsBSPPNrTWqA5h7h54solrw5L4IypJT_o" + func TestAuthJWTCookie(t *testing.T) { a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour), PermissionChecker: &mockUserPermissions{}} @@ -54,6 +56,14 @@ func TestAuthJWTCookie(t *testing.T) { resp, err = client.Do(req) require.NoError(t, err) assert.Equal(t, 201, resp.StatusCode, "token expired and refreshed") + + req, err = http.NewRequest("GET", server.URL+"/auth", nil) + require.Nil(t, err) + req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtNoUser, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false}) + req.Header.Add("X-XSRF-TOKEN", "random id") + resp, err = client.Do(req) + require.NoError(t, err) + assert.Equal(t, 401, resp.StatusCode, "no user info in the token") } func TestAuthJWTHeader(t *testing.T) { From a71907fd8e32e883ec739d4e9618b7c7bc8acf90 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 12 Sep 2018 00:28:00 -0500 Subject: [PATCH 88/93] reduce frome AdminStore to KeyStore in Authenticator --- backend/app/rest/auth/auth.go | 12 ++++++++---- backend/app/rest/auth/auth_test.go | 2 +- backend/app/rest/auth/jwt.go | 5 ----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index 758924ab..98cbc72f 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -9,18 +9,22 @@ import ( "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/admin" ) // Authenticator is top level auth object providing middlewares type Authenticator struct { JWTService *JWT Providers []Provider - AdminStore admin.Store + KeysStore KeyStore DevPasswd string PermissionChecker PermissionChecker } +// KeyStore defines sub-interface for consumers needed just a key +type KeyStore interface { + Key(siteID string) (key string, err error) +} + var devUser = store.User{ ID: "dev", Name: "developer one", @@ -112,14 +116,14 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { } func (a *Authenticator) checkSecretKey(r *http.Request) bool { - if a.AdminStore == nil { + if a.KeysStore == nil { return false } siteID := r.URL.Query().Get("site") secret := r.URL.Query().Get("secret") - skey, err := a.AdminStore.Key(siteID) + skey, err := a.KeysStore.Key(siteID) if err != nil { return false } diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index c69df4d9..5350bc10 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -223,7 +223,7 @@ func TestAdminRequired(t *testing.T) { } func TestAuthWithSecret(t *testing.T) { - a := Authenticator{DevPasswd: "123456", AdminStore: admin.NewStaticKeyStore("secretkey")} + a := Authenticator{DevPasswd: "123456", KeysStore: admin.NewStaticKeyStore("secretkey")} router := chi.NewRouter() router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) diff --git a/backend/app/rest/auth/jwt.go b/backend/app/rest/auth/jwt.go index a95f2934..0326ce48 100644 --- a/backend/app/rest/auth/jwt.go +++ b/backend/app/rest/auth/jwt.go @@ -42,11 +42,6 @@ const jwtHeaderKey = "X-JWT" const xsrfCookieName = "XSRF-TOKEN" const xsrfHeaderKey = "X-XSRF-TOKEN" -// KeyStore defines sub-interface for consumers needed just a key -type KeyStore interface { - Key(siteID string) (key string, err error) -} - // NewJWT makes JWT service func NewJWT(keyStore KeyStore, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT { res := JWT{ From dc59fc9f7bc9cae4721201519363b5784bed8f3c Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 12 Sep 2018 00:34:36 -0500 Subject: [PATCH 89/93] adjust test for key store --- backend/app/cmd/server.go | 2 +- backend/app/cmd/server_test.go | 2 +- backend/app/rest/api/migrator_test.go | 2 +- backend/app/rest/api/rest_test.go | 2 +- backend/app/rest/auth/auth.go | 6 +++--- backend/app/rest/auth/auth_test.go | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index bc1a34ac..4f868e8d 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -225,7 +225,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { SharedSecret: s.SharedSecret, Authenticator: auth.Authenticator{ JWTService: jwtService, - AdminStore: adminStore, + KeyStore: adminStore, Providers: authProviders, DevPasswd: s.DevPasswd, PermissionChecker: dataService, diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 67eddf62..dacaf21f 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -45,7 +45,7 @@ func TestServerApp(t *testing.T) { body, _ = ioutil.ReadAll(resp.Body) t.Log(string(body)) - assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminStore.Email(""), "default admin email") + assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email") app.Wait() } diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index 6771d37a..c192bd94 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -165,7 +165,7 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *http a := auth.Authenticator{ DevPasswd: "password", Providers: nil, - AdminStore: adminStore, + KeyStore: adminStore, JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour), } routes := svc.withRoutes(chi.NewRouter().With(a.Auth(true)).With(a.AdminOnly)) diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 666caed5..8717b50e 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -104,7 +104,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { Authenticator: auth.Authenticator{ DevPasswd: "password", Providers: nil, - AdminStore: adminStore, + KeyStore: adminStore, JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour), }, Cache: &cache.Nop{}, diff --git a/backend/app/rest/auth/auth.go b/backend/app/rest/auth/auth.go index 98cbc72f..61b0e35c 100644 --- a/backend/app/rest/auth/auth.go +++ b/backend/app/rest/auth/auth.go @@ -15,7 +15,7 @@ import ( type Authenticator struct { JWTService *JWT Providers []Provider - KeysStore KeyStore + KeyStore KeyStore DevPasswd string PermissionChecker PermissionChecker } @@ -116,14 +116,14 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler { } func (a *Authenticator) checkSecretKey(r *http.Request) bool { - if a.KeysStore == nil { + if a.KeyStore == nil { return false } siteID := r.URL.Query().Get("site") secret := r.URL.Query().Get("secret") - skey, err := a.KeysStore.Key(siteID) + skey, err := a.KeyStore.Key(siteID) if err != nil { return false } diff --git a/backend/app/rest/auth/auth_test.go b/backend/app/rest/auth/auth_test.go index 5350bc10..7b901835 100644 --- a/backend/app/rest/auth/auth_test.go +++ b/backend/app/rest/auth/auth_test.go @@ -223,7 +223,7 @@ func TestAdminRequired(t *testing.T) { } func TestAuthWithSecret(t *testing.T) { - a := Authenticator{DevPasswd: "123456", KeysStore: admin.NewStaticKeyStore("secretkey")} + a := Authenticator{DevPasswd: "123456", KeyStore: admin.NewStaticKeyStore("secretkey")} router := chi.NewRouter() router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(201) From c70e060d220a63bd097c89218dbf4279610cfd07 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 14 Sep 2018 20:33:40 -0500 Subject: [PATCH 90/93] Feature/avatar_bolt (#205) * add bolt implementation of avatar store * include bolt avatar store in main's test * change docs with new avatart params * lint: single line declaration * separate bucket for avatars metas * fix comments * test for bolt params * closable avatar store * fix dbl-close --- README.md | 79 +++++++------- backend/app/cmd/avatar.go | 16 +++ backend/app/cmd/avatar_test.go | 12 +++ backend/app/cmd/server.go | 18 +++- backend/app/cmd/server_test.go | 4 +- backend/app/store/avatar/bolt.go | 142 ++++++++++++++++++++++++++ backend/app/store/avatar/bolt_test.go | 98 ++++++++++++++++++ backend/app/store/avatar/gridfs.go | 5 + backend/app/store/avatar/localfs.go | 5 + backend/app/store/avatar/store.go | 2 +- 10 files changed, 337 insertions(+), 44 deletions(-) create mode 100644 backend/app/store/avatar/bolt.go create mode 100644 backend/app/store/avatar/bolt_test.go diff --git a/README.md b/README.md index 004f68c4..8a7d17a9 100644 --- a/README.md +++ b/README.md @@ -82,46 +82,45 @@ _this is the recommended way to run remark42_ #### Parameters -| Command line | Environment | Default | Description | -| ------------------ | ------------------ | --------------------- | ---------------------------------------------- | -| url | REMARK_URL | | url to remark42 server, _required_ | -| secret | SECRET | | secret key, _required_ | -| site | SITE | `remark` | site name(s), _multi_ | -| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `mongo` | -| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory | -| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout | -| store.mongo.url | STORE_MONGO_URL | | mongo url for data store | -| store.mongo.db | STORE_MONGO_DB | | mongo db for data store | -| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ | -| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email | -| backup | BACKUP_PATH | `./var/backup` | backups location | -| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep | -| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited | -| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited | -| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited | -| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs` or `mongo` | -| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store | -| avatar.mongo.url | AVATAR_MONGO_URL | | mongo url for avatar store | -| avatar.mongo.db | AVATAR_MONGO_DB | | mongo db for avatar store | -| avatar.rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save | -| max-comment | MAX_COMMENT_SIZE | 2048 | comment's size limit | -| auth.ttl.jwt | AUTH_TTL_JWT | 5m | jwt TTL | -| auth.ttl.cookie | AUTH_TTL_COOKIE | 200h | cookie TTL | -| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID | -| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret | -| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID | -| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret | -| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID | -| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret | -| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID | -| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret | -| auth.dev | AUTH_DEV | false | local oauth2 server, development mode only | -| low-score | LOW_SCORE | `-5` | low score threshold | -| critical-score | CRITICAL_SCORE | `-10` | critical score threshold | -| edit-time | EDIT_TIME | `5m` | edit window | -| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images | -| dbg | DEBUG | `false` | debug mode | -| dev-passwd | DEV_PASSWD | | password for `dev` user | +| Command line | Environment | Default | Description | +| ------------------ | ------------------ | --------------------- | ------------------------------------------------ | +| url | REMARK_URL | | url to remark42 server, _required_ | +| secret | SECRET | | secret key, _required_ | +| site | SITE | `remark` | site name(s), _multi_ | +| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `mongo` | +| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory | +| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout | +| mongo.url | MONGO_URL | | mongo url for all stores using mongodb | +| mongo.db | MONGO_DB | | mongo database | +| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ | +| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email | +| backup | BACKUP_PATH | `./var/backup` | backups location | +| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep | +| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited | +| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited | +| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited | +| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, 'bolt`, or `mongo` | +| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store | +| avatar.bolt.file | AVATAR_BOLT_FILE | `./var/avatars.db` | file name for `bolt` store | +| avatar.rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save | +| auth.ttl.jwt | AUTH_TTL_JWT | 5m | jwt TTL | +| auth.ttl.cookie | AUTH_TTL_COOKIE | 200h | cookie TTL | +| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID | +| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret | +| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID | +| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret | +| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID | +| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret | +| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID | +| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret | +| auth.dev | AUTH_DEV | false | local oauth2 server, development mode only | +| max-comment | MAX_COMMENT_SIZE | 2048 | comment's size limit | +| low-score | LOW_SCORE | `-5` | low score threshold | +| critical-score | CRITICAL_SCORE | `-10` | critical score threshold | +| edit-time | EDIT_TIME | `5m` | edit window | +| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images | +| dbg | DEBUG | `false` | debug mode | +| dev-passwd | DEV_PASSWD | | password for `dev` user | * command line parameters are long form `--=value`, i.e. `--site=https://demo.remark42.com` * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...` diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go index c4cd506d..7321a09f 100644 --- a/backend/app/cmd/avatar.go +++ b/backend/app/cmd/avatar.go @@ -2,8 +2,10 @@ package cmd import ( "log" + "path" "time" + "github.com/coreos/bbolt" "github.com/go-pkgz/mongo" "github.com/pkg/errors" @@ -54,11 +56,20 @@ func (ac *AvatarCommand) Execute(args []string) error { if err != nil { return err } + + if err = dst.Close(); err != nil { + log.Printf("[WARN] failed to close dst store %s", ac.AvatarDst.Type) + } + if err = src.Close(); err != nil { + log.Printf("[WARN] failed to close src store %s", ac.AvatarSrc.Type) + } + log.Printf("[INFO] completed, migrated avatars = %d", count) return nil } func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) { + log.Printf("[DEBUG] make avatar store, type=%s", gr.Type) switch gr.Type { case "fs": if err := makeDirs(gr.FS.Path); err != nil { @@ -72,6 +83,11 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) { } conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "") return avatar.NewGridFS(conn, gr.RszLmt), nil + case "bolt": + if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil { + return nil, err + } + return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{}, gr.RszLmt) } return nil, errors.Errorf("unsupported avatar store type %s", gr.Type) } diff --git a/backend/app/cmd/avatar_test.go b/backend/app/cmd/avatar_test.go index 5362ae08..c7c62c91 100644 --- a/backend/app/cmd/avatar_test.go +++ b/backend/app/cmd/avatar_test.go @@ -22,6 +22,7 @@ func TestAvatar_Execute(t *testing.T) { } defer os.RemoveAll("/tmp/ava-test") + // from fs to mongo cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}} cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) p := flags.NewParser(&cmd, flags.Default) @@ -31,6 +32,17 @@ func TestAvatar_Execute(t *testing.T) { err = cmd.Execute(nil) assert.NoError(t, err) + // from fs to bolt + cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}} + cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt", + "--dst.bolt.file=/tmp/ava-test.db"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) + + // failed cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}} cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) p = flags.NewParser(&cmd, flags.Default) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 4f868e8d..13cd9817 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "os/signal" + "path" "strings" "syscall" "time" @@ -80,10 +81,13 @@ type StoreGroup struct { // AvatarGroup defines options group for avatar params type AvatarGroup struct { - Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"` + Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"` FS struct { Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` } `group:"fs" namespace:"fs" env-namespace:"FS"` + Bolt struct { + File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"` + } `group:"bolt" namespace:"bolt" env-namespace:"bolt"` RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` } @@ -120,6 +124,7 @@ type serverApp struct { exporter migrator.Exporter devAuth *auth.DevAuthServer dataService *service.DataStore + avatarStore avatar.Store terminated chan struct{} } @@ -247,6 +252,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { exporter: exporter, devAuth: devAuth, dataService: dataService, + avatarStore: avatarStore, terminated: make(chan struct{}), }, nil } @@ -265,7 +271,10 @@ func (a *serverApp) run(ctx context.Context) error { a.devAuth.Shutdown() } if e := a.dataService.Close(); e != nil { - log.Printf("[WARN] failed to close store, %s", e) + log.Printf("[WARN] failed to close data store, %s", e) + } + if e := a.avatarStore.Close(); e != nil { + log.Printf("[WARN] failed to close avatar store, %s", e) } }() @@ -340,6 +349,11 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { } conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") return avatar.NewGridFS(conn, s.Avatar.RszLmt), nil + case "bolt": + if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil { + return nil, err + } + return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{}, s.Avatar.RszLmt) } return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type) } diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index dacaf21f..053be5c5 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -202,7 +202,9 @@ func TestServerApp_MainSignal(t *testing.T) { s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) p := flags.NewParser(&s, flags.Default) - args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", "--port=18100"} + args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt", + "--avatar.bolt.file=/tmp/ava-test.db", "--port=18100"} + defer os.Remove("/tmp/ava-test.db") _, err := p.ParseArgs(args) require.Nil(t, err) err = s.Execute(args) diff --git a/backend/app/store/avatar/bolt.go b/backend/app/store/avatar/bolt.go new file mode 100644 index 00000000..90ed197f --- /dev/null +++ b/backend/app/store/avatar/bolt.go @@ -0,0 +1,142 @@ +package avatar + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "io" + "io/ioutil" + "log" + + "github.com/coreos/bbolt" + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store" +) + +// BoltDB implements avatar store with bolt +// using separate db (file) with "avatars" bucket to keep image bin and "metas" bucket to keep sha1 +// avatarID (base file name) used as a key +type BoltDB struct { + fileName string // full path to boltdb + resizeLimit int + db *bolt.DB +} + +const avatarsBktName = "avatars" +const metasBktName = "metas" + +// NewBoltDB makes bolt avatar store +func NewBoltDB(fileName string, options bolt.Options, resizeLimit int) (*BoltDB, error) { + db, err := bolt.Open(fileName, 0600, &options) + if err != nil { + return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName) + } + err = db.Update(func(tx *bolt.Tx) error { + if _, e := tx.CreateBucketIfNotExists([]byte(avatarsBktName)); e != nil { + return errors.Wrapf(e, "failed to create top level bucket %s", avatarsBktName) + } + _, e := tx.CreateBucketIfNotExists([]byte(metasBktName)) + return errors.Wrapf(e, "failed to create top metas bucket %s", metasBktName) + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName) + } + return &BoltDB{db: db, fileName: fileName, resizeLimit: resizeLimit}, nil +} + +// Put avatar to bolt, key by avatarID +func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error) { + id := encodeID(userID) + + // Trying to resize avatar. + if reader = resize(reader, b.resizeLimit); reader == nil { + return "", errors.New("avatar resize reader is nil") + } + + avatarID := id + imgSfx + err = b.db.Update(func(tx *bolt.Tx) error { + buf := &bytes.Buffer{} + if _, err = io.Copy(buf, reader); err != nil { + return errors.Wrapf(err, "can't read avatar %s", avatarID) + } + + if err = tx.Bucket([]byte(avatarsBktName)).Put([]byte(avatarID), buf.Bytes()); err != nil { + return errors.Wrapf(err, "can't put to bucket with %s", avatarID) + } + return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), id))) + }) + return avatarID, err +} + +// Get avatar reader for avatar id.image, avatarID used as the direct key +func (b *BoltDB) Get(avatarID string) (reader io.ReadCloser, size int, err error) { + buf := &bytes.Buffer{} + err = b.db.View(func(tx *bolt.Tx) error { + data := tx.Bucket([]byte(avatarsBktName)).Get([]byte(avatarID)) + if data == nil { + return errors.Errorf("can't load avatar %s", avatarID) + } + size, err = buf.Write(data) + return errors.Wrapf(err, "failed to write for %s", avatarID) + }) + return ioutil.NopCloser(buf), size, err +} + +// ID returns a fingerprint of the avatar content. +func (b *BoltDB) ID(avatarID string) (id string) { + data := []byte{} + err := b.db.View(func(tx *bolt.Tx) error { + data = tx.Bucket([]byte(metasBktName)).Get([]byte(avatarID)) + if data == nil { + return errors.Errorf("can't load avatar's id for %s", avatarID) + } + return nil + }) + if err != nil { + log.Printf("[DEBUG] can't get avatar info '%s', %s", avatarID, err) + return store.EncodeID(avatarID) + } + return string(data) +} + +// Remove avatar from bolt +func (b *BoltDB) Remove(avatarID string) (err error) { + return b.db.Update(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(avatarsBktName)) + if bkt.Get([]byte(avatarID)) == nil { + return errors.Errorf("avatar key not found, %s", avatarID) + } + if err = tx.Bucket([]byte(avatarsBktName)).Delete([]byte(avatarID)); err != nil { + return errors.Wrapf(err, "can't delete avatar object %s", avatarID) + } + return errors.Wrapf(tx.Bucket([]byte(metasBktName)).Delete([]byte(avatarID)), + "can't delete meta object %s", avatarID) + }) +} + +// List all avatars (ids) from metas bucket +// note: id includes .image suffix +func (b *BoltDB) List() (ids []string, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + return tx.Bucket([]byte(metasBktName)).ForEach(func(k, _ []byte) error { + ids = append(ids, string(k)) + return nil + }) + }) + return ids, errors.Wrap(err, "failed to list") +} + +// Close bolt store +func (b *BoltDB) Close() error { + return errors.Wrapf(b.db.Close(), "failed to close %s", b.fileName) +} + +func (b *BoltDB) sha1(data []byte, avatarID string) (id string) { + h := sha1.New() + if _, err := h.Write(data); err != nil { + log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err) + return store.EncodeID(avatarID) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/backend/app/store/avatar/bolt_test.go b/backend/app/store/avatar/bolt_test.go new file mode 100644 index 00000000..4db0376e --- /dev/null +++ b/backend/app/store/avatar/bolt_test.go @@ -0,0 +1,98 @@ +package avatar + +import ( + "io/ioutil" + "os" + "sort" + "strings" + "testing" + + "github.com/coreos/bbolt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testDb = "/tmp/test-remark-avatars.db" + +func TestBoltDB_PutAndGet(t *testing.T) { + var b Store = prepBoltStore(t) + defer func() { + assert.Nil(t, b.Close()) + os.Remove(testDb) + }() + + avatar, err := b.Put("user1", strings.NewReader("some picture bin data")) + require.Nil(t, err) + assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) + + rd, size, err := b.Get(avatar) + require.Nil(t, err) + assert.Equal(t, 21, size) + data, err := ioutil.ReadAll(rd) + require.Nil(t, err) + assert.Equal(t, "some picture bin data", string(data)) + + _, _, err = b.Get("bad avatar") + assert.NotNil(t, err) + + assert.Equal(t, "fddae9ce556712a6ece0e8951a6e7a05c51ed6bf", b.ID(avatar)) + assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", b.ID("aaaa"), "no data, encode avatar id") + + l, err := b.List() + require.Nil(t, err) + assert.Equal(t, 1, len(l)) + assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0]) +} + +func TestBoltDB_Remove(t *testing.T) { + b := prepBoltStore(t) + defer func() { + assert.Nil(t, b.Close()) + os.Remove(testDb) + }() + + assert.NotNil(t, b.Remove("no-such-thing.image")) + + avatar, err := b.Put("user1", strings.NewReader("some picture bin data")) + require.Nil(t, err) + assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) + assert.NoError(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one") + assert.NotNil(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed") +} + +func TestBoltDB_List(t *testing.T) { + b := prepBoltStore(t) + defer func() { + assert.Nil(t, b.Close()) + os.Remove(testDb) + }() + + // write some avatars + _, err := b.Put("user1", strings.NewReader("some picture bin data 1")) + require.Nil(t, err) + _, err = b.Put("user2", strings.NewReader("some picture bin data 2")) + require.Nil(t, err) + _, err = b.Put("user3", strings.NewReader("some picture bin data 3")) + require.Nil(t, err) + + l, err := b.List() + assert.NoError(t, err) + assert.Equal(t, 3, len(l), "3 avatars listed") + sort.Strings(l) + assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) + + r, size, err := b.Get("0b7f849446d3383546d15a480966084442cd2193.image") + assert.Nil(t, err) + assert.Equal(t, 23, size) + data, err := ioutil.ReadAll(r) + assert.Nil(t, err) + assert.Equal(t, "some picture bin data 3", string(data)) +} + +// makes new boltdb, put two records +func prepBoltStore(t *testing.T) *BoltDB { + os.Remove(testDb) + boltStore, err := NewBoltDB(testDb, bolt.Options{}, 0) + require.Nil(t, err) + return boltStore +} diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index 79c925f6..6a20253e 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -118,3 +118,8 @@ func (gf *GridFS) List() (ids []string, err error) { } return ids, errors.Wrap(err, "can't list avatars") } + +// Close gridfs does nothing but satisfies interface +func (gf *GridFS) Close() error { + return nil +} diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go index dd70acad..9b34ac3d 100644 --- a/backend/app/store/avatar/localfs.go +++ b/backend/app/store/avatar/localfs.go @@ -113,6 +113,11 @@ func (fs *LocalFS) List() (ids []string, err error) { return ids, errors.Wrap(err, "can't list avatars") } +// Close gridfs does nothing but satisfies interface +func (fs *LocalFS) Close() error { + return nil +} + // get location (directory) for user id by adding partition to final path in order to keep files // in different subdirectories and avoid too many files in a single place. // the end result is a full path like this - /tmp/avatars.test/92 diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go index add4c83c..a3876207 100644 --- a/backend/app/store/avatar/store.go +++ b/backend/app/store/avatar/store.go @@ -32,7 +32,7 @@ type Store interface { ID(avatarID string) (id string) // unique id of stored avatar's data Remove(avatarID string) error // remove avatar data List() (ids []string, err error) // list all avatar ids - + Close() error } // Migrate avatars between stores From e610657e3679e7db70bff39597b3c41f886ed898 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 15 Sep 2018 12:33:04 -0500 Subject: [PATCH 91/93] better comments for avatar store --- backend/app/cmd/avatar.go | 3 ++- backend/app/store/avatar/bolt.go | 16 +++++++++------- backend/app/store/avatar/bolt_test.go | 3 ++- backend/app/store/avatar/store.go | 5 +++-- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go index 7321a09f..52174fe5 100644 --- a/backend/app/cmd/avatar.go +++ b/backend/app/cmd/avatar.go @@ -13,7 +13,8 @@ import ( ) // AvatarCommand set of flags and command for avatar migration -// it converts all avatarts from src.type to dst.type +// it converts all avatars from src.type to dst.type. +// Note: it is possible to run migration for the same types (src = dst) in order to resize all avatars. type AvatarCommand struct { AvatarSrc AvatarGroup `group:"src" namespace:"src"` AvatarDst AvatarGroup `group:"dst" namespace:"dst"` diff --git a/backend/app/store/avatar/bolt.go b/backend/app/store/avatar/bolt.go index 90ed197f..826a1ded 100644 --- a/backend/app/store/avatar/bolt.go +++ b/backend/app/store/avatar/bolt.go @@ -15,8 +15,8 @@ import ( ) // BoltDB implements avatar store with bolt -// using separate db (file) with "avatars" bucket to keep image bin and "metas" bucket to keep sha1 -// avatarID (base file name) used as a key +// using separate db (file) with "avatars" bucket to keep image bin and "metas" bucket +// to keep sha1 of picture. avatarID (base file name) used as a key for both. type BoltDB struct { fileName string // full path to boltdb resizeLimit int @@ -45,7 +45,7 @@ func NewBoltDB(fileName string, options bolt.Options, resizeLimit int) (*BoltDB, return &BoltDB{db: db, fileName: fileName, resizeLimit: resizeLimit}, nil } -// Put avatar to bolt, key by avatarID +// Put avatar to bolt, key by avatarID. Trying to resize image and lso calculates sha1 of the file for ID func func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error) { id := encodeID(userID) @@ -64,7 +64,8 @@ func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error) if err = tx.Bucket([]byte(avatarsBktName)).Put([]byte(avatarID), buf.Bytes()); err != nil { return errors.Wrapf(err, "can't put to bucket with %s", avatarID) } - return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), id))) + // store sha1 of the image + return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), avatarID))) }) return avatarID, err } @@ -87,16 +88,17 @@ func (b *BoltDB) Get(avatarID string) (reader io.ReadCloser, size int, err error func (b *BoltDB) ID(avatarID string) (id string) { data := []byte{} err := b.db.View(func(tx *bolt.Tx) error { - data = tx.Bucket([]byte(metasBktName)).Get([]byte(avatarID)) - if data == nil { + if data = tx.Bucket([]byte(metasBktName)).Get([]byte(avatarID)); data == nil { return errors.Errorf("can't load avatar's id for %s", avatarID) } return nil }) - if err != nil { + + if err != nil { // failed to get ID, use encoded avatarID log.Printf("[DEBUG] can't get avatar info '%s', %s", avatarID, err) return store.EncodeID(avatarID) } + return string(data) } diff --git a/backend/app/store/avatar/bolt_test.go b/backend/app/store/avatar/bolt_test.go index 4db0376e..b9ddac23 100644 --- a/backend/app/store/avatar/bolt_test.go +++ b/backend/app/store/avatar/bolt_test.go @@ -35,8 +35,9 @@ func TestBoltDB_PutAndGet(t *testing.T) { _, _, err = b.Get("bad avatar") assert.NotNil(t, err) + // check IDs assert.Equal(t, "fddae9ce556712a6ece0e8951a6e7a05c51ed6bf", b.ID(avatar)) - assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", b.ID("aaaa"), "no data, encode avatar id") + assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", b.ID("aaaa"), "no data, encoded avatar id") l, err := b.List() require.Nil(t, err) diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go index a3876207..eb9a3aed 100644 --- a/backend/app/store/avatar/store.go +++ b/backend/app/store/avatar/store.go @@ -1,4 +1,5 @@ -// Package avatar defines store interface and implements local (fs) and gridfs (mongo) stores. +// Package avatar defines store interface and implements local (fs), gridfs (mongo) and boltdb stores. +// package avatar //go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp store_mock.go" @@ -32,7 +33,7 @@ type Store interface { ID(avatarID string) (id string) // unique id of stored avatar's data Remove(avatarID string) error // remove avatar data List() (ids []string, err error) // list all avatar ids - Close() error + Close() error // close store } // Migrate avatars between stores From be00380b5ac35869918826cb1223f267b334d83e Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 15 Sep 2018 14:18:28 -0500 Subject: [PATCH 92/93] lint: spell fix --- backend/app/rest/auth/dev_provider.go | 2 +- backend/app/store/admin/admin_test.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/backend/app/rest/auth/dev_provider.go b/backend/app/rest/auth/dev_provider.go index 2e465d4e..1c07ffe7 100644 --- a/backend/app/rest/auth/dev_provider.go +++ b/backend/app/rest/auth/dev_provider.go @@ -51,7 +51,7 @@ func (d *DevAuthServer) Run() { switch { case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"): - // first time it will be called without username and will ask for onw + // first time it will be called without username and will ask for one if !d.nonInteractive && (r.ParseForm() != nil || r.Form.Get("username") == "") { if _, err = w.Write([]byte(fmt.Sprintf(devUserForm, r.URL.RawQuery))); err != nil { log.Printf("[WARN] can't write, %s", err) diff --git a/backend/app/store/admin/admin_test.go b/backend/app/store/admin/admin_test.go index ebbbec1e..27f4b8ea 100644 --- a/backend/app/store/admin/admin_test.go +++ b/backend/app/store/admin/admin_test.go @@ -40,10 +40,7 @@ func TestMongoStore_Get(t *testing.T) { if e1 := coll.Insert(recs[0]); e1 != nil { return e1 } - if e2 := coll.Insert(recs[1]); e2 != nil { - return e2 - } - return nil + return coll.Insert(recs[1]) }) require.NoError(t, err) From 72aeadcfd1ae9676f5112dc57f64486ac233985c Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 16 Sep 2018 12:51:09 -0500 Subject: [PATCH 93/93] remove legacy Avatart interface from engine --- backend/app/store/engine/engine.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go index 507efa8c..8d150808 100644 --- a/backend/app/store/engine/engine.go +++ b/backend/app/store/engine/engine.go @@ -3,7 +3,6 @@ package engine import ( - "io" "sort" "strings" "time" @@ -56,13 +55,6 @@ type Admin interface { IsVerified(siteID string, userID string) bool // check verified status } -// Avatar defines interface for avatar store -type Avatar interface { - Put(userID string, reader io.Reader) (avatar string, err error) - Get(avatar string) (reader io.ReadCloser, size int, err error) - ID(avatar string) (id string) -} - const ( // limits lastLimit = 1000