From 3fa4ab418d04e2d8cb19defdcfd8b656150f2a1c Mon Sep 17 00:00:00 2001 From: Eugene Date: Sun, 24 Dec 2017 02:43:56 -0600 Subject: [PATCH] start importer from disqus --- .gitignore | 4 +- app/migrator/disqus.go | 98 +++++++++++++++++++++++++++++++ app/migrator/disqus_test.go | 111 ++++++++++++++++++++++++++++++++++++ app/migrator/importer.go | 8 +++ app/store/bolt.go | 3 +- app/store/store.go | 2 +- 6 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 app/migrator/disqus.go create mode 100644 app/migrator/disqus_test.go create mode 100644 app/migrator/importer.go diff --git a/.gitignore b/.gitignore index 675772cf..be46f784 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ target /logs/ /target/ -/var/ \ No newline at end of file +/var/ +debug +debug.test diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go new file mode 100644 index 00000000..678724ea --- /dev/null +++ b/app/migrator/disqus.go @@ -0,0 +1,98 @@ +package migrator + +import ( + "encoding/xml" + "io" + "io/ioutil" + "log" + "time" + + "github.com/pkg/errors" + "github.com/umputun/remark/app/store" +) + +// Disqus implements Importer from disqus xml +type Disqus struct { + DataStore store.Interface +} + +type disqusXML struct { + Threads []disqusThread `xml:"thread"` + Comments []disqusComment `xml:"post"` +} + +type disqusThread struct { + UID string `xml:"id,attr"` + Forum string `xml:"forum"` + Link string `xml:"link"` + Title string `xml:"title"` + Message string `xml:"message"` + CreateAt time.Time `xml:"createdAt"` + AuthorNmae string `xml:"author>name"` + AuthorEmail string `xml:"author>email"` + Anonymous bool `xml:"author>isAnonymous"` + IP string `xml:"ipAddress"` + Closed bool `xml:"isClosed"` + Deleted bool `xml:"isDeleted"` +} + +type disqusComment struct { + UID string `xml:"id,attr"` + ID string `xml:"id"` + Message string `xml:"message"` + CreatedAt time.Time `xml:"createdAt"` + AuthorEmail string `xml:"author>email"` + AuthorName string `xml:"author>name"` + AuthorUserName string `xml:"author>username"` + IP string `xml:"ipAddress"` + Tid uid `xml:"thread"` + Pid uid `xml:"parent"` +} + +type uid struct { + Val string `xml:"id,attr"` +} + +// Import from disqus and save to store +func (d Disqus) Import(r io.Reader) (err error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return errors.Wrap(err, "failed to read data") + } + dxml := disqusXML{} + if err = xml.Unmarshal(data, &dxml); err != nil { + return errors.Wrap(err, "can't unmarshal disqus xml") + } + + failed := 0 + for _, c := range d.convert(dxml) { + if _, err = d.DataStore.Create(c); err != nil { + failed++ + } + } + + if failed > 0 { + return errors.Errorf("failed to save %d comments", failed) + } + return nil +} + +func (d Disqus) convert(dxml disqusXML) (comments []store.Comment) { + postsMap := map[string]string{} // tid:url + log.Printf("[DEBUG] convert %d posts, %d comments", len(dxml.Threads), len(dxml.Comments)) + for _, thread := range dxml.Threads { + postsMap[thread.UID] = thread.Link + } + for _, comment := range dxml.Comments { + c := store.Comment{ + ID: comment.ID, + Locator: store.Locator{URL: postsMap[comment.Tid.Val]}, + User: store.User{ID: comment.AuthorUserName, Name: comment.AuthorName, IP: comment.IP}, + Text: comment.Message, + Timestamp: comment.CreatedAt, + ParentID: comment.Pid.Val, + } + comments = append(comments, c) + } + return comments +} diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go new file mode 100644 index 00000000..76571756 --- /dev/null +++ b/app/migrator/disqus_test.go @@ -0,0 +1,111 @@ +package migrator + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestA(t *testing.T) { + d := Disqus{} + err := d.Import(strings.NewReader(xmlTest)) + assert.Nil(t, err) +} + +var xmlTest = ` + + + + + + radiot + + http://radio-t.umputun.com/2011/03/229_8880.html + Радио-Т: Радио-Т 229 + + 2011-03-07T20:46:25Z + + umputun@gmail.com + Umputun + false + umputun + + 98.212.28.115 + false + false + + + http://www.radio-t.com/p/2011/03/05/podcast-229/ + radiot + + https://radio-t.com/p/2011/03/05/podcast-229/ + Радио-Т: Радио-Т 229 + + 2011-03-07T21:17:17Z + + umputun@gmail.com + Umputun + false + umputun + + 80.250.214.235 + true + false + + + + 3565798471341011339 + + Недавно посмотрел фильм про Джобса и Aple. Тот он еще был перец. http://alex-bestbusiness.com/

]]> + + 2011-08-31T15:16:29Z + false + false + + + Alexander Puzatykh + false + facebook-1787732238 + + 178.234.205.125 + + + + + 3029154520436241933 + + Microsoft показал проводник Windows 8 с ленточным интерфейсом.

http://blogs.msdn.com/b/b8/... 

]]> + + 2011-08-31T17:44:22Z + false + false + + mihail.merkulov@gmail.com + mikhailmerkulov + false + mikhailmerkulov + + 195.234.75.139 + + + + + 6580890074280459209 + + Google App Engine скоро выходит из превью статуса.

Сейчас письмо пришло от гугла.

Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.
Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).
В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.
Всем кто включит биллинг до 31 октября, обещают 50 баксов :)

]]> + + 2011-08-31T22:48:43Z + false + false + + unikier@gmail.com + Dmitry Shapoval + false + google-74b9e7568ef6860e93862c5d7752b657 + + 89.113.25.139 + + + +` diff --git a/app/migrator/importer.go b/app/migrator/importer.go new file mode 100644 index 00000000..341b5d4b --- /dev/null +++ b/app/migrator/importer.go @@ -0,0 +1,8 @@ +package migrator + +import "io" + +// Importer defines interface to convert posts from external sources +type Importer interface { + Import(r io.Reader) error +} diff --git a/app/store/bolt.go b/app/store/bolt.go index f5e8030b..13f2cebe 100644 --- a/app/store/bolt.go +++ b/app/store/bolt.go @@ -4,11 +4,10 @@ import ( "encoding/json" "fmt" "log" + "sort" "strings" "time" - "sort" - "github.com/boltdb/bolt" "github.com/pkg/errors" ) diff --git a/app/store/store.go b/app/store/store.go index a9e69ef0..2880b9c5 100644 --- a/app/store/store.go +++ b/app/store/store.go @@ -8,7 +8,7 @@ import ( "time" ) -// Comment represents a single comment with optinal reference to its parent +// Comment represents a single comment with optional reference to its parent type Comment struct { ID string `json:"id"` ParentID string `json:"pid"`