mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove import cmd, add import test

Browse Source
This commit is contained in:
Umputun
2018-04-30 02:14:14 -05:00
parent 9b9479367a
commit 15095a9151
7 changed files with 81 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.go
+37 -61
View File
@@ -33,30 +33,22 @@ var opts struct {
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var" description:"backups location"`
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
ServerCommand struct {
SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"`
StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"`
SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"`
StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"`
GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"`
GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"`
GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"`
GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"`
FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"`
FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"`
DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"`
DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"`
GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"`
GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"`
GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"`
GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"`
FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"`
FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"`
DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"`
DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"`
AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"`
DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"`
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
} `command:"server" description:"run server"`
ImportCommand struct {
Provider string `long:"provider" default:"disqus" description:"provider type"`
SiteID string `long:"site" default:"remark" description:"site ID"`
InputFile string `long:"file" default:"disqus.xml" description:"input file"`
} `command:"import" description:"import comments from external sources"`
AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"`
DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"`
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
}
var revision = "unknown"
@@ -71,33 +63,19 @@ func main() {
setupLog(opts.Dbg)
log.Print("[INFO] started remark")
if err := makeDirs(opts.BoltPath, opts.ServerCommand.SessionStore, opts.BackupLocation, opts.ServerCommand.AvatarStore); err != nil {
if err := makeDirs(opts.BoltPath, opts.SessionStore, opts.BackupLocation, opts.AvatarStore); err != nil {
log.Fatalf("[ERROR] can't create directories, %+v", err)
}
dataStore := makeBoltStore(opts.Sites)
if p.Active != nil && p.Command.Find("import") == p.Active {
// import mode
params := migrator.ImportParams{
CommentCreator: dataStore,
InputFile: opts.ImportCommand.InputFile,
Provider: opts.ImportCommand.Provider,
SiteID: opts.ImportCommand.SiteID,
}
if err := migrator.ImportComments(params); err != nil {
log.Fatalf("[ERROR] failed to import, %+v", err)
}
return
}
if opts.DevPasswd != "" {
log.Printf("[WARN] running in dev mode")
}
dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.ServerCommand.StoreKey}
dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.StoreKey}
sessionStore := func() sessions.Store {
sess := sessions.NewFilesystemStore(opts.ServerCommand.SessionStore, []byte(opts.ServerCommand.StoreKey))
sess := sessions.NewFilesystemStore(opts.SessionStore, []byte(opts.StoreKey))
sess.Options.HttpOnly = true
sess.Options.Secure = true
sess.Options.MaxAge = 3600 * 24 * 365
@@ -105,24 +83,23 @@ func main() {
return sess
}()
exporter := &migrator.Remark{CommentFinder: dataStore}
nativeMigrator := migrator.Remark{CommentFinder: dataStore}
avatarProxy := &auth.AvatarProxy{
StorePath: opts.ServerCommand.AvatarStore,
StorePath: opts.AvatarStore,
RoutePath: "/api/v1/avatar",
RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
DefaultAvatar: opts.ServerCommand.DefaultAvatar,
DefaultAvatar: opts.DefaultAvatar,
}
activateBackup(exporter)
activateBackup(&nativeMigrator)
srv := api.Rest{
Version: revision,
DataService: dataService,
Exporter: exporter,
Importer: &migrator.Remark{CommentCreator: dataStore},
Disqus: &migrator.Disqus{CommentCreator: dataStore},
WebRoot: opts.ServerCommand.WebRoot,
Version: revision,
DataService: dataService,
NativeMigrator: nativeMigrator,
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
WebRoot: opts.WebRoot,
Authenticator: auth.Authenticator{
Admins: opts.Admins,
SessionStore: sessionStore,
@@ -132,7 +109,7 @@ func main() {
},
Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn),
}
srv.Run(opts.ServerCommand.Port)
srv.Run(opts.Port)
}
// activateBackup runs background backups for each site
@@ -203,19 +180,17 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx
}
}
srvOpts := opts.ServerCommand
if srvOpts.GoogleCID != "" && srvOpts.GoogleCSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(srvOpts.GoogleCID, srvOpts.GoogleCSEC)))
if opts.GoogleCID != "" && opts.GoogleCSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(opts.GoogleCID, opts.GoogleCSEC)))
}
if srvOpts.GithubCID != "" && srvOpts.GithubCSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(srvOpts.GithubCID, srvOpts.GithubCSEC)))
if opts.GithubCID != "" && opts.GithubCSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(opts.GithubCID, opts.GithubCSEC)))
}
if srvOpts.FacebookCID != "" && srvOpts.FacebookCSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(srvOpts.FacebookCID, srvOpts.FacebookCSEC)))
if opts.FacebookCID != "" && opts.FacebookCSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(opts.FacebookCID, opts.FacebookCSEC)))
}
if srvOpts.DisqusCID != "" && srvOpts.DisqusCSEC != "" {
providers = append(providers, auth.NewDisqus(makeParams(srvOpts.DisqusCID, srvOpts.DisqusCSEC)))
if opts.DisqusCID != "" && opts.DisqusCSEC != "" {
providers = append(providers, auth.NewDisqus(makeParams(opts.DisqusCID, opts.DisqusCSEC)))
}
if len(providers) == 0 {
log.Printf("[WARN] no auth providers defined")
@@ -226,6 +201,7 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx
// post-flush callback invoked by cache after each flush in async way
func postFlushFn() {
// list of heavy urls for pre-heating on cache change
urls := []string{
"http://localhost:%d/api/v1/list?site=%s",
"http://localhost:%d/api/v1/last/50?site=%s",
@@ -233,7 +209,7 @@ func postFlushFn() {
for _, site := range opts.Sites {
for _, u := range urls {
resp, err := http.Get(fmt.Sprintf(u, opts.ServerCommand.Port, site))
resp, err := http.Get(fmt.Sprintf(u, opts.Port, site))
if err != nil {
log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err)
return
app/rest/api/admin.go
+6 -6
View File
@@ -19,9 +19,8 @@ import (
// admin provides router for all requests available for admin users only
type admin struct {
dataService store.Service
exporterNative migrator.Exporter
importerNative migrator.Importer
importerDisqus migrator.Importer
nativeMigrator migrator.Remark
disqusImporter migrator.Importer
cache rest.LoadingCache
defAvatarURL string
}
@@ -108,7 +107,7 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
writer = gzip.NewWriter(w)
}
if err := a.exporterNative.Export(writer, siteID); err != nil {
if err := a.nativeMigrator.Export(writer, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
}
}
@@ -117,13 +116,14 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
// imports comments from post body.
func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
importer := a.importerNative
var importer migrator.Importer = &a.nativeMigrator
if r.URL.Query().Get("provider") == "disqus" {
importer = a.importerDisqus
importer = a.disqusImporter
}
if err := importer.Import(r.Body, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
return
}
a.cache.Flush()
}
app/rest/api/admin_test.go
+29
View File
@@ -160,3 +160,32 @@ func TestAdmin_Export(t *testing.T) {
assert.Equal(t, 2, strings.Count(body, "\"text\""))
t.Logf("%s", body)
}
func TestAdmin_Import(t *testing.T) {
srv, port := prep(t)
assert.NotNil(t, srv)
defer cleanup(srv)
// add 2 initial comments, will be deleted by import
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
c2 := store.Comment{Text: "test test #2",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
addComment(t, c1, port)
addComment(t, c2, port)
r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
port), "application/json", r)
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, code := get(t, fmt.Sprintf("http://127.0.0.1:%d/api/v1/list?site=radio-t", port))
assert.Equal(t, 200, code)
pi := []store.PostInfo{}
err = json.Unmarshal([]byte(body), &pi)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 1}, {URL: "https://radio-t.com/blah1", Count: 1}}, pi)
}
app/rest/api/rest.go
+4 -6
View File
@@ -35,9 +35,8 @@ type Rest struct {
Cache rest.LoadingCache
WebRoot string
Exporter migrator.Exporter
Importer migrator.Importer
Disqus migrator.Importer
NativeMigrator migrator.Remark
DisqusImporter migrator.Importer
httpServer *http.Server
amdminService admin
@@ -53,9 +52,8 @@ func (s *Rest) Run(port int) {
s.amdminService = admin{
dataService: s.DataService,
exporterNative: s.Exporter,
importerNative: s.Importer,
importerDisqus: s.Disqus,
nativeMigrator: s.NativeMigrator,
disqusImporter: s.DisqusImporter,
cache: s.Cache,
defAvatarURL: s.Authenticator.AvatarProxy.Default(),
}
app/rest/api/rest_test.go
+4 -3
View File
@@ -421,9 +421,10 @@ func prep(t *testing.T) (srv *Rest, port int) {
AvatarProxy: &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"},
Admins: []string{"a1", "a2"},
},
Exporter: &migrator.Remark{CommentFinder: dataStore},
Cache: &mockCache{},
WebRoot: "/tmp",
NativeMigrator: migrator.Remark{CommentFinder: dataStore, CommentCreator: dataStore},
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
Cache: &mockCache{},
WebRoot: "/tmp",
}
ioutil.WriteFile(testHTML, []byte("some html"), 0700)
remark.rest
-1
View File
@@ -107,4 +107,3 @@ GET {{host}}/api/v1/avatar/blah
### ping
GET {{host}}/ping
start.sh
+1 -1
View File
@@ -12,4 +12,4 @@ cp -nv /srv/web/remark.image /srv/var/avatars/remark.image
echo "start remark42 server"
/sbin/su-exec ${USER} /srv/remark server
/sbin/su-exec ${USER} /srv/remark