package migrator import ( "bytes" "encoding/json" "io" log "github.com/go-pkgz/lgr" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/service" ) const natvieVersion = 1 // Native implements exporter and importer for internal store format // {"version": 1, comments:[{...}\n,{}], meta: {meta}} // each comments starts from the new line type Native struct { DataStore Store } type meta struct { Version int `json:"version"` Users []service.UserMetaData `json:"users"` Posts []service.PostMetaData `json:"posts"` } // Export all comments to writer as json strings. Each comment is one string, separated by "\n" // The final file is a valid json func (n *Native) Export(w io.Writer, siteID string) (size int, err error) { if err = n.exportMeta(siteID, w); err != nil { return 0, errors.Wrapf(err, "failed to export meta for site %s", siteID) } topics, err := n.DataStore.List(siteID, 0, 0) if err != nil { return 0, err } log.Printf("[DEBUG] exporting %d topics", len(topics)) commentsCount := 0 for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction topic := topics[i] comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") if err != nil { return commentsCount, e } for _, comment := range comments { buf := &bytes.Buffer{} enc := json.NewEncoder(buf) enc.SetEscapeHTML(false) if err = enc.Encode(comment); err != nil { return commentsCount, errors.Wrapf(err, "can't marshal %v", comments) } if _, err = w.Write(buf.Bytes()); err != nil { return commentsCount, errors.Wrap(err, "can't write comment data") } commentsCount++ } } log.Printf("[DEBUG] exported %d comments", commentsCount) return commentsCount, nil } // exportMeta appends user and post metas to exported stream func (n *Native) exportMeta(siteID string, w io.Writer) (err error) { m := meta{Version: natvieVersion} m.Users, m.Posts, err = n.DataStore.Metas(siteID) if err != nil { return errors.Wrap(err, "can't get meta") } if err := json.NewEncoder(w).Encode(m); err != nil { return errors.Wrap(err, "can't encode meta") } return nil } // Import comments from json strings produced by Remark.Export func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { m := meta{} dec := json.NewDecoder(reader) if err = dec.Decode(&m); err != nil { return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID) } if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version return 0, errors.Errorf("unexpected import file version %d", m.Version) } if err = n.DataStore.DeleteAll(siteID); err != nil { return 0, err } failed := 0 total, comments := 0, 0 for { comment := store.Comment{} err = dec.Decode(&comment) if err == io.EOF { break } total++ if err != nil { failed++ continue } if _, err = n.DataStore.Create(comment); err != nil { failed++ log.Printf("[WARN] can't write %+v to store, %s", comment, err) continue } comments++ if comments%1000 == 0 { log.Printf("[DEBUG] imported %d comments", comments) } } if failed > 0 { return comments, errors.Errorf("failed to save %d comments", failed) } log.Printf("[INFO] imported %d comments from %d records", comments, total) err = n.DataStore.SetMetas(siteID, m.Users, m.Posts) return comments, err }