mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move import to local dedicated server

Browse Source
This commit is contained in:
Umputun
2018-04-30 03:13:45 -05:00
parent 15095a9151
commit 4f8cabf69e
9 changed files with 200 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+32 -34
View File
@@ -23,34 +23,34 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
* copy provided `docker-compose.yml` and customize for your needs
* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
* prepare user id for container ``export USER=`id -u $USER` ``
* prepare user id for container `` export USER=`id -u $USER` ``
* make sure you **don't keep** `DEV=true` for any non-development deployments
* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
* alternatively compile from sources - `docker-compose build`
#### Parameters
| Command line | Environment | Default | Multi | Scope | Description |
| --------------- | -------------------- | ---------------------- | ----- | ------ | ------------------------------- |
| --url | REMARK_URL | `https://remark42.com` | no | all | url to remark server |
| --bolt | BOLTDB_PATH | `/tmp` | no | all | path to data directory |
| --site | SITE | `remark` | yes | server | site name(s) |
| --admin | ADMIN | | yes | server | admin names (list of user ids) |
| --backup | BACKUP_PATH | `/tmp` | no | server | backups location |
| --max-back | MAX_BACKUP_FILES | `10` | no | server | max backup files to keep |
| --session | SESSION_STORE | `/tmp` | no | server | path to session store directory |
| --store-key | STORE_KEY | `secure-store-key` | no | server | session store encryption key |
| --google-cid | REMARK_GOOGLE_CID | | no | server | Google OAuth client ID |
| --google-csec | REMARK_GOOGLE_CSEC | | no | server | Google OAuth client secret |
| --facebook-cid | REMARK_FACEBOOK_CID | | no | server | Facebook OAuth client ID |
| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | server | Facebook OAuth client secret |
| --github-cid | REMARK_GITHUB_CID | | no | server | Github OAuth client ID |
| --github-csec | REMARK_GITHUB_CSEC | | no | server | Github OAuth client secret |
| --provider | | `disqus` | no | import | provider type for import |
| --site | | `remark` | no | import | site ID |
| --file | | `disqus.xml` | no | import | import file |
| --dbg | DEBUG | `false` | no | all | debug mode |
| --dev-password | DEV_PASSWD | | no | all | password for `dev` user |
| Command line | Environment | Default | Multi | Description |
| --------------- | -------------------- | ---------------------- | ----- | ------------------------------- |
| --url | REMARK_URL | `https://remark42.com` | no | url to remark server |
| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory |
| --site | SITE | `remark` | yes | site name(s) |
| --admin | ADMIN | | yes | admin names (list of user ids) |
| --backup | BACKUP_PATH | `/tmp` | no | backups location |
| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep |
| --session | SESSION_STORE | `/tmp` | no | path to session store directory |
| --store-key | STORE_KEY | `secure-store-key` | no | session store encryption key |
| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID |
| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret |
| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID |
| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret |
| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID |
| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret |
| --provider | | `disqus` | no | provider type for import |
| --site | | `remark` | no | site ID |
| --file | | `disqus.xml` | no | import file |
| --dbg | DEBUG | `false` | no | debug mode |
| --dev-password | DEV_PASSWD | | no | password for `dev` user |
_all multi parameters separated by `,`_
@@ -101,10 +101,8 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
#### Initial import from Disqus
1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
2. Move this file to your remark42 host and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
3. Stop remark42 containers if started, i.e. `docker-compose stop`
4. Run import command - `docker-compose run --rm remark /srv/remark import --file=<disqus-export-name>.xml --site=<your site id>`
5. Start remark42 containers `docker-compose up -d`
2. Move this file to your remark42 host within `.var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
3. Run import command - `docker-compose run remark /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
### Frontend
@@ -112,20 +110,20 @@ Frontend part is building automatically along with backend if you use `docker-co
For manual building:
- install [Node.js 8](https://nodejs.org/en/) or higher;
- run `npm install` inside `./web`;
- run `npm run build` there;
- result files will be saved in `./web/public`.
* install [Node.js 8](https://nodejs.org/en/) or higher;
* run `npm install` inside `./web`;
* run `npm run build` there;
* result files will be saved in `./web/public`.
For development mode use `npm start` instead of `npm run build`.
In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`.
URLs for development:
- `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
- `localhost:8080/dev.html` — page with embedded script from local folder;
- `localhost:8080/last-comments.html` — page with embedded script for last comments;
- `localhost:8080/counter.html` — page with embedded script for counter with examples.
* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
* `localhost:8080/dev.html` — page with embedded script from local folder;
* `localhost:8080/last-comments.html` — page with embedded script for last comments;
* `localhost:8080/counter.html` — page with embedded script for counter with examples.
## API
app/main.go
+17 -9
View File
@@ -83,7 +83,18 @@ func main() {
return sess
}()
nativeMigrator := migrator.Remark{CommentFinder: dataStore}
exporter := migrator.Remark{CommentFinder: dataStore}
cache := rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn)
activateBackup(&exporter)
importSrv := api.Import{
Version: revision,
Cache: cache,
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
}
go importSrv.Run(opts.Port + 1)
avatarProxy := &auth.AvatarProxy{
StorePath: opts.AvatarStore,
@@ -92,14 +103,11 @@ func main() {
DefaultAvatar: opts.DefaultAvatar,
}
activateBackup(&nativeMigrator)
srv := api.Rest{
Version: revision,
DataService: dataService,
NativeMigrator: nativeMigrator,
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
WebRoot: opts.WebRoot,
Version: revision,
DataService: dataService,
Exporter: &exporter,
WebRoot: opts.WebRoot,
Authenticator: auth.Authenticator{
Admins: opts.Admins,
SessionStore: sessionStore,
@@ -107,7 +115,7 @@ func main() {
AvatarProxy: avatarProxy,
DevPasswd: opts.DevPasswd,
},
Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn),
Cache: cache,
}
srv.Run(opts.Port)
}
app/rest/api/admin.go
+6 -23
View File
@@ -18,11 +18,10 @@ import (
// admin provides router for all requests available for admin users only
type admin struct {
dataService store.Service
nativeMigrator migrator.Remark
disqusImporter migrator.Importer
cache rest.LoadingCache
defAvatarURL string
dataService store.Service
exporter migrator.Exporter
cache rest.LoadingCache
defAvatarURL string
}
func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
@@ -31,7 +30,7 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
router.Delete("/comment/{id}", a.deleteCommentCtrl)
router.Put("/user/{userid}", a.setBlockCtrl)
router.Get("/export", a.exportCtrl)
router.Post("/import", a.importCtrl)
router.Put("/pin/{id}", a.setPinCtrl)
router.Get("/blocked", a.blockedUsersCtrl)
return router
@@ -107,27 +106,11 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
writer = gzip.NewWriter(w)
}
if err := a.nativeMigrator.Export(writer, siteID); err != nil {
if err := a.exporter.Export(writer, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
}
}
// POST /import?site=site-id&provider=disqus|remark
// imports comments from post body.
func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
var importer migrator.Importer = &a.nativeMigrator
if r.URL.Query().Get("provider") == "disqus" {
importer = a.disqusImporter
}
if err := importer.Import(r.Body, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
return
}
a.cache.Flush()
}
func (a *admin) checkBlocked(siteID string, user store.User) bool {
return a.dataService.IsBlocked(siteID, user.ID)
}
app/rest/api/admin_test.go
-29
View File
@@ -160,32 +160,3 @@ func TestAdmin_Export(t *testing.T) {
assert.Equal(t, 2, strings.Count(body, "\"text\""))
t.Logf("%s", body)
}
func TestAdmin_Import(t *testing.T) {
srv, port := prep(t)
assert.NotNil(t, srv)
defer cleanup(srv)
// add 2 initial comments, will be deleted by import
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
c2 := store.Comment{Text: "test test #2",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
addComment(t, c1, port)
addComment(t, c2, port)
r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
port), "application/json", r)
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, code := get(t, fmt.Sprintf("http://127.0.0.1:%d/api/v1/list?site=radio-t", port))
assert.Equal(t, 200, code)
pi := []store.PostInfo{}
err = json.Unmarshal([]byte(body), &pi)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 1}, {URL: "https://radio-t.com/blah1", Count: 1}}, pi)
}
app/rest/api/import.go
+59
View File
@@ -0,0 +1,59 @@
package api
import (
"fmt"
"log"
"net/http"
"time"
"github.com/didip/tollbooth"
"github.com/didip/tollbooth_chi"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
"github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/rest"
)
// Import rest runs on unexposed port and available for local requests only
type Import struct {
Version string
Cache rest.LoadingCache
NativeImporter migrator.Importer
DisqusImporter migrator.Importer
httpServer *http.Server
}
// Run the listener and request's router, activate rest server
// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way
func (s *Import) Run(port int) {
log.Printf("[INFO] activate import server on port %d", port)
router := chi.NewRouter()
router.Use(middleware.RealIP, Recoverer)
router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
router.Use(AppInfo("remark42-importer", s.Version), Ping, Logger(LogAll))
router.Post("/api/v1/admin/import", s.importCtrl)
s.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router}
err := s.httpServer.ListenAndServe()
log.Printf("[WARN] http server terminated, %s", err)
}
// POST /import?site=site-id&provider=disqus|remark
// imports comments from post body.
func (s *Import) importCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
importer := s.NativeImporter
if r.URL.Query().Get("provider") == "disqus" {
importer = s.DisqusImporter
}
if err := importer.Import(r.Body, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
return
}
s.Cache.Flush()
}
app/rest/api/import_test.go
+63
View File
@@ -0,0 +1,63 @@
package api
import (
"context"
"fmt"
"math/rand"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/store"
)
func TestImport(t *testing.T) {
srv, port := prepImportSrv(t)
assert.NotNil(t, srv)
defer cleanupImportSrv(srv)
r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
port), "application/json", r)
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
}
func prepImportSrv(t *testing.T) (srv *Import, port int) {
dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
srv = &Import{
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
Cache: &mockCache{},
}
portSetCh := make(chan bool)
go func() {
port = rand.Intn(50000) + 1025
portSetCh <- true
srv.Run(port)
}()
<-portSetCh
time.Sleep(100 * time.Millisecond)
return srv, port
}
func cleanupImportSrv(srv *Import) {
srv.httpServer.Close()
srv.httpServer.Shutdown(context.Background())
os.Remove(testDb)
}
app/rest/api/rest.go
+6 -8
View File
@@ -35,8 +35,7 @@ type Rest struct {
Cache rest.LoadingCache
WebRoot string
NativeMigrator migrator.Remark
DisqusImporter migrator.Importer
Exporter migrator.Exporter
httpServer *http.Server
amdminService admin
@@ -44,18 +43,17 @@ type Rest struct {
// Run the lister and request's router, activate rest server
func (s *Rest) Run(port int) {
log.Print("[INFO] activate rest server")
log.Printf("[INFO] activate rest server on port %d", port)
if len(s.Authenticator.Admins) > 0 {
log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins)
}
s.amdminService = admin{
dataService: s.DataService,
nativeMigrator: s.NativeMigrator,
disqusImporter: s.DisqusImporter,
cache: s.Cache,
defAvatarURL: s.Authenticator.AvatarProxy.Default(),
dataService: s.DataService,
exporter: s.Exporter,
cache: s.Cache,
defAvatarURL: s.Authenticator.AvatarProxy.Default(),
}
router := chi.NewRouter()
app/rest/api/rest_test.go
+13 -2
View File
@@ -421,21 +421,32 @@ func prep(t *testing.T) (srv *Rest, port int) {
AvatarProxy: &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"},
Admins: []string{"a1", "a2"},
},
NativeMigrator: migrator.Remark{CommentFinder: dataStore, CommentCreator: dataStore},
Exporter: &migrator.Remark{CommentFinder: dataStore},
Cache: &mockCache{},
WebRoot: "/tmp",
}
importSrv := &Import{
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
Cache: &mockCache{},
WebRoot: "/tmp",
}
ioutil.WriteFile(testHTML, []byte("some html"), 0700)
portSetCh := make(chan bool)
go func() {
port = rand.Intn(50000) + 1025
portSetCh <- true
srv.Run(port)
}()
<-portSetCh
go func() {
importSrv.Run(port + 1)
}()
time.Sleep(100 * time.Millisecond)
return srv, port
}
import-disqus.sh
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
echo "import disqus file $1 to site $2"
curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 http://127.0.0.1:8081/api/v1/admin/import?site=remark &
provider=disqus