diff --git a/app/main.go b/app/main.go index 73a65a0f..28cc2410 100644 --- a/app/main.go +++ b/app/main.go @@ -9,6 +9,8 @@ import ( "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" + "github.com/pkg/errors" + "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest" "github.com/umputun/remark/app/rest/auth" "github.com/umputun/remark/app/store" @@ -29,28 +31,40 @@ var opts struct { Admins []string `long:"admin" env:"ADMIN" default:"umputun@gmail.com" description:"admin(s) names" env-delim:","` DevMode bool `long:"dev" env:"DEV" description:"development mode, no auth enforced"` Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` + + ImportCommand struct { + Provider string `long:"provider" default:"disqus" description:"provider type"` + SiteID string `long:"site" default:"site" description:"site ID"` + InputFile string `long:"file" default:"disqus.xml" description:"input file"` + } `command:"import" description:"import comments from external sources"` } var revision = "unknown" func main() { fmt.Printf("remark %s\n", revision) - if _, err := flags.Parse(&opts); err != nil { - log.Fatal(err) + p := flags.NewParser(&opts, flags.Default) + if _, e := p.ParseArgs(os.Args[1:]); e != nil { + os.Exit(1) } + setupLog(opts.Dbg) log.Print("[INFO] started remark") - if opts.DevMode { - log.Printf("[WARN] running in dev mode, no auth!") - } - dataStore, err := store.NewBoltDB(opts.DBFile) if err != nil { log.Fatalf("[ERROR] can't initialize data store, %+v", err) } + if p.Active != nil && p.Command.Find("import") == p.Active { + if err := importComments(dataStore); err != nil { + log.Fatalf("[ERROR] failed to import, %+v", err) + } + return + } + sessionStore := sessions.NewFilesystemStore(opts.SessionStore, []byte(opts.StoreKey)) + srv := rest.Server{ Version: revision, Store: dataStore, @@ -71,9 +85,26 @@ func main() { }), } + if opts.DevMode { + log.Printf("[WARN] running in dev mode, no auth!") + } + srv.Run() } +func importComments(dataStore store.Interface) error { + log.Printf("[INFO] import from %s (%s) to %s", + opts.ImportCommand.InputFile, opts.ImportCommand.Provider, opts.ImportCommand.SiteID) + importer := migrator.Disqus{DataStore: dataStore} + + fh, err := os.Open(opts.ImportCommand.InputFile) + if err != nil { + return errors.Wrapf(err, "can't open import file %s", opts.ImportCommand.InputFile) + } + defer fh.Close() + return importer.Import(fh, opts.ImportCommand.SiteID) +} + func setupLog(dbg bool) { filter := &logutils.LevelFilter{ Levels: []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR"}, diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go index 2d80104f..7a288ca2 100644 --- a/app/migrator/disqus.go +++ b/app/migrator/disqus.go @@ -3,7 +3,6 @@ package migrator import ( "encoding/xml" "io" - "io/ioutil" "log" "time" @@ -59,15 +58,6 @@ type uid struct { // Import from disqus and save to store func (d *Disqus) Import(r io.Reader, siteID string) (err error) { - data, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrap(err, "failed to read data") - } - dxml := disqusXML{} - if err = xml.Unmarshal(data, &dxml); err != nil { - return errors.Wrap(err, "can't unmarshal disqus xml") - } - commentsCh := d.convert(r, siteID) failed := 0 for c := range commentsCh { @@ -89,6 +79,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { decoder := xml.NewDecoder(r) commentsCh := make(chan store.Comment) + inpThreads, inpComments := 0, 0 go func() { commentsCount := 0 for { @@ -100,12 +91,15 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { switch se := t.(type) { case xml.StartElement: if se.Name.Local == "thread" { + inpThreads++ thread := disqusThread{} if err := decoder.DecodeElement(&thread, &se); err == nil { postsMap[thread.UID] = thread.Link } + continue } if se.Name.Local == "post" { + inpComments++ comment := disqusComment{} if err := decoder.DecodeElement(&comment, &se); err != nil { continue @@ -123,11 +117,15 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { } commentsCh <- c commentsCount++ + if commentsCount%1000 == 0 { + log.Printf("[DEBUG] imported %d comments", commentsCount) + } } + } } close(commentsCh) - log.Printf("[DEBUG] converted %d posts, %d comments", len(postsMap), commentsCount) + log.Printf("[INFO] converted %d posts with %d comments from disqus %d/%d", len(postsMap), commentsCount, inpThreads, inpComments) }() return commentsCh diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go index 3a4b00ab..98754492 100644 --- a/app/migrator/disqus_test.go +++ b/app/migrator/disqus_test.go @@ -38,10 +38,15 @@ func TestDisqus_Convert(t *testing.T) { assert.Equal(t, exp0, res[0]) } -var xmlTest = ` - +var xmlTest = ` + + radiot + General + true + + radiot