91 lines
2.1 KiB
Go
91 lines
2.1 KiB
Go
package migrator
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/umputun/remark/app/store"
|
|
)
|
|
|
|
// Remark implements exporter and importer for internal store
|
|
type Remark struct {
|
|
DataStore store.Interface
|
|
}
|
|
|
|
// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
|
|
func (r *Remark) Export(w io.Writer, siteID string) error {
|
|
topics, err := r.DataStore.List(siteID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Printf("[DEBUG] exporting %d topics", len(topics))
|
|
|
|
commentsCount := 0
|
|
for _, topic := range topics {
|
|
comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, comment := range comments {
|
|
|
|
buf := &bytes.Buffer{}
|
|
enc := json.NewEncoder(buf)
|
|
enc.SetEscapeHTML(false)
|
|
|
|
if err := enc.Encode(comment); err != nil {
|
|
return errors.Wrapf(err, "can't marshal %v", comments)
|
|
}
|
|
data := buf.Bytes()
|
|
if _, err := w.Write(data); err != nil {
|
|
return errors.Wrap(err, "can't write comment data")
|
|
}
|
|
commentsCount++
|
|
}
|
|
}
|
|
log.Printf("[DEBUG] exported %d comments", commentsCount)
|
|
return nil
|
|
}
|
|
|
|
// Import comments from json strings produced by Remark.Export
|
|
func (r *Remark) Import(reader io.Reader, siteID string) error {
|
|
failed := 0
|
|
total, comments := 0, 0
|
|
scanner := bufio.NewScanner(reader)
|
|
for scanner.Scan() {
|
|
rec := scanner.Bytes()
|
|
if len(rec) < 2 {
|
|
continue
|
|
}
|
|
total++
|
|
comment := store.Comment{}
|
|
if err := json.Unmarshal(rec, &comment); err != nil {
|
|
failed++
|
|
log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err)
|
|
continue
|
|
}
|
|
if _, err := r.DataStore.Create(comment); err != nil {
|
|
failed++
|
|
log.Printf("[WARN] can't write %+v to store, %s", comment, err)
|
|
continue
|
|
}
|
|
comments++
|
|
if comments%1000 == 0 {
|
|
log.Printf("[DEBUG] imported %d comments", comments)
|
|
}
|
|
}
|
|
if scanner.Err() != nil {
|
|
return errors.Wrap(scanner.Err(), "error in scan")
|
|
}
|
|
if failed > 0 {
|
|
return errors.Errorf("failed to save %d comments", failed)
|
|
}
|
|
log.Printf("[INFO] imported %d comments from %d records", comments, total)
|
|
return nil
|
|
}
|