integrate imported as sub-command

app/main.go

@@ -9,6 +9,8 @@ import (
 	"github.com/hashicorp/logutils"
 	"github.com/jessevdk/go-flags"
 
+	"github.com/pkg/errors"
+	"github.com/umputun/remark/app/migrator"
 	"github.com/umputun/remark/app/rest"
 	"github.com/umputun/remark/app/rest/auth"
 	"github.com/umputun/remark/app/store"
@@ -29,28 +31,40 @@ var opts struct {
 	Admins  []string `long:"admin" env:"ADMIN" default:"umputun@gmail.com" description:"admin(s) names" env-delim:","`
 	DevMode bool     `long:"dev" env:"DEV" description:"development mode, no auth enforced"`
 	Dbg     bool     `long:"dbg" env:"DEBUG" description:"debug mode"`
+
+	ImportCommand struct {
+		Provider  string `long:"provider" default:"disqus" description:"provider type"`
+		SiteID    string `long:"site" default:"site" description:"site ID"`
+		InputFile string `long:"file" default:"disqus.xml" description:"input file"`
+	} `command:"import" description:"import comments from external sources"`
 }
 
 var revision = "unknown"
 
 func main() {
 	fmt.Printf("remark %s\n", revision)
-	if _, err := flags.Parse(&opts); err != nil {
-		log.Fatal(err)
+	p := flags.NewParser(&opts, flags.Default)
+	if _, e := p.ParseArgs(os.Args[1:]); e != nil {
+		os.Exit(1)
 	}
+
 	setupLog(opts.Dbg)
 	log.Print("[INFO] started remark")
 
-	if opts.DevMode {
-		log.Printf("[WARN] running in dev mode, no auth!")
-	}
-
 	dataStore, err := store.NewBoltDB(opts.DBFile)
 	if err != nil {
 		log.Fatalf("[ERROR] can't initialize data store, %+v", err)
 	}
 
+	if p.Active != nil && p.Command.Find("import") == p.Active {
+		if err := importComments(dataStore); err != nil {
+			log.Fatalf("[ERROR] failed to import, %+v", err)
+		}
+		return
+	}
+
 	sessionStore := sessions.NewFilesystemStore(opts.SessionStore, []byte(opts.StoreKey))
+
 	srv := rest.Server{
 		Version:      revision,
 		Store:        dataStore,
@@ -71,9 +85,26 @@ func main() {
 		}),
 	}
 
+	if opts.DevMode {
+		log.Printf("[WARN] running in dev mode, no auth!")
+	}
+
 	srv.Run()
 }
 
+func importComments(dataStore store.Interface) error {
+	log.Printf("[INFO] import from %s (%s) to %s",
+		opts.ImportCommand.InputFile, opts.ImportCommand.Provider, opts.ImportCommand.SiteID)
+	importer := migrator.Disqus{DataStore: dataStore}
+
+	fh, err := os.Open(opts.ImportCommand.InputFile)
+	if err != nil {
+		return errors.Wrapf(err, "can't open import file %s", opts.ImportCommand.InputFile)
+	}
+	defer fh.Close()
+	return importer.Import(fh, opts.ImportCommand.SiteID)
+}
+
 func setupLog(dbg bool) {
 	filter := &logutils.LevelFilter{
 		Levels:   []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR"},

app/migrator/disqus.go

@@ -3,7 +3,6 @@ package migrator
 import (
 	"encoding/xml"
 	"io"
-	"io/ioutil"
 	"log"
 	"time"
 
@@ -59,15 +58,6 @@ type uid struct {
 // Import from disqus and save to store
 func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
 
-	data, err := ioutil.ReadAll(r)
-	if err != nil {
-		return errors.Wrap(err, "failed to read data")
-	}
-	dxml := disqusXML{}
-	if err = xml.Unmarshal(data, &dxml); err != nil {
-		return errors.Wrap(err, "can't unmarshal disqus xml")
-	}
-
 	commentsCh := d.convert(r, siteID)
 	failed := 0
 	for c := range commentsCh {
@@ -89,6 +79,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 	decoder := xml.NewDecoder(r)
 	commentsCh := make(chan store.Comment)
 
+	inpThreads, inpComments := 0, 0
 	go func() {
 		commentsCount := 0
 		for {
@@ -100,12 +91,15 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 			switch se := t.(type) {
 			case xml.StartElement:
 				if se.Name.Local == "thread" {
+					inpThreads++
 					thread := disqusThread{}
 					if err := decoder.DecodeElement(&thread, &se); err == nil {
 						postsMap[thread.UID] = thread.Link
 					}
+					continue
 				}
 				if se.Name.Local == "post" {
+					inpComments++
 					comment := disqusComment{}
 					if err := decoder.DecodeElement(&comment, &se); err != nil {
 						continue
@@ -123,11 +117,15 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 					}
 					commentsCh <- c
 					commentsCount++
+					if commentsCount%1000 == 0 {
+						log.Printf("[DEBUG] imported %d comments", commentsCount)
+					}
 				}
+
 			}
 		}
 		close(commentsCh)
-		log.Printf("[DEBUG] converted %d posts, %d comments", len(postsMap), commentsCount)
+		log.Printf("[INFO] converted %d posts with %d comments from disqus %d/%d", len(postsMap), commentsCount, inpThreads, inpComments)
 	}()
 
 	return commentsCh

app/migrator/disqus_test.go

@@ -38,10 +38,15 @@ func TestDisqus_Convert(t *testing.T) {
 	assert.Equal(t, exp0, res[0])
 }
 
-var xmlTest = `
-<?xml version="1.0" encoding="utf-8"?>
+var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
 <disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
 
+	<category dsq:id="707279">
+		<forum>radiot</forum>
+		<title>General</title>
+		<isDefault>true</isDefault>
+	</category>
+
 	<thread dsq:id="247918464">
 		<id/>
 		<forum>radiot</forum>