package migrator import ( "encoding/xml" "fmt" "io" "strings" "time" log "github.com/go-pkgz/lgr" "github.com/umputun/remark42/backend/app/store" ) // Disqus implements Importer from disqus xml type Disqus struct { DataStore Store } type disqusThread struct { UID string `xml:"id,attr"` Forum string `xml:"forum"` Link string `xml:"link"` Title string `xml:"title"` Message string `xml:"message"` CreateAt time.Time `xml:"createdAt"` AuthorName string `xml:"author>name"` AuthorEmail string `xml:"author>email"` IP string `xml:"ipAddress"` Anonymous bool `xml:"author>isAnonymous"` Closed bool `xml:"isClosed"` Deleted bool `xml:"isDeleted"` } type disqusComment struct { UID string `xml:"id,attr"` ID string `xml:"id"` Message string `xml:"message"` CreatedAt time.Time `xml:"createdAt"` AuthorEmail string `xml:"author>email"` AuthorName string `xml:"author>name"` AuthorUserName string `xml:"author>username"` IP string `xml:"ipAddress"` Tid uid `xml:"thread"` Pid uid `xml:"parent"` IsSpam bool `xml:"isSpam"` Deleted bool `xml:"isDeleted"` } type uid struct { Val string `xml:"id,attr"` } // Import from disqus and save to store func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) { if e := d.DataStore.DeleteAll(siteID); e != nil { return 0, e } commentsCh := d.convert(r, siteID) failed, passed := 0, 0 for c := range commentsCh { if _, err = d.DataStore.Create(c); err != nil { failed++ continue } passed++ } if failed > 0 { err = fmt.Errorf("failed to save %d comments", failed) if passed == 0 { err = fmt.Errorf("import failed") } } log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID) return passed, err } // convert disqus stream (xml) from reader and fill channel of comments. // runs async and closes channel on completion. func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { postsMap := map[string]string{} // tid:url decoder := xml.NewDecoder(r) commentsCh := make(chan store.Comment) stats := struct { inpThreads, inpComments int commentsCount, spamComments int failedThreads, failedPosts int deletedComments, skippedComments int }{} go func() { for { t, err := decoder.Token() if t == nil || err != nil { break } if se, ok := t.(xml.StartElement); ok { if se.Name.Local == "thread" { stats.inpThreads++ thread := disqusThread{} if err = decoder.DecodeElement(&thread, &se); err != nil { log.Printf("[WARN] can't decode disqus thread, %s", err) stats.failedThreads++ continue } if thread.Deleted { continue } postsMap[thread.UID] = thread.Link continue } if se.Name.Local == "post" { stats.inpComments++ comment := disqusComment{} if err = decoder.DecodeElement(&comment, &se); err != nil { log.Printf("[WARN] can't decode disqus comment, %s", err) stats.failedPosts++ continue } if comment.Deleted { stats.deletedComments++ continue } if comment.IsSpam { stats.spamComments++ continue } url, ok := postsMap[comment.Tid.Val] if !ok { stats.skippedComments++ continue } c := store.Comment{ ID: comment.UID, Locator: store.Locator{URL: url, SiteID: siteID}, User: store.User{ ID: "disqus_" + store.EncodeID(comment.AuthorUserName), Name: comment.AuthorName, IP: comment.IP, }, Text: d.cleanText(comment.Message), Timestamp: comment.CreatedAt, ParentID: comment.Pid.Val, Imported: true, } if comment.AuthorUserName == "" { // empty comment.AuthorUserName from disqus c.User.ID = "disqus_" + store.EncodeID(c.User.Name) } if c.ID == "" { // no comment.UID c.ID = comment.ID } commentsCh <- c stats.commentsCount++ if stats.commentsCount%1000 == 0 { log.Printf("[DEBUG] processed %d comments", stats.commentsCount) } } } } close(commentsCh) log.Printf("[INFO] converted %d posts, %+v", len(postsMap), stats) }() return commentsCh } func (*Disqus) cleanText(text string) string { text = strings.TrimSpace(text) text = strings.ReplaceAll(text, "\n", "") text = strings.ReplaceAll(text, "\t", "") return text }