From 9838aaac7021a102fba013a79db29171cb96c095 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 6 May 2021 18:03:15 -0500 Subject: [PATCH] Disqus fix (#989) --- backend/app/migrator/disqus.go | 27 ++- backend/app/migrator/disqus_test.go | 220 ++++++------------ backend/app/migrator/migrator_test.go | 6 +- .../migrator/testdata/disqus-deleted-post.xml | 150 ++++++++++++ .../testdata/disqus-deleted-thread.xml | 150 ++++++++++++ backend/app/migrator/testdata/disqus.xml | 150 ++++++++++++ 6 files changed, 540 insertions(+), 163 deletions(-) create mode 100644 backend/app/migrator/testdata/disqus-deleted-post.xml create mode 100644 backend/app/migrator/testdata/disqus-deleted-thread.xml create mode 100644 backend/app/migrator/testdata/disqus.xml diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go index 0dab4b23..48c18446 100644 --- a/backend/app/migrator/disqus.go +++ b/backend/app/migrator/disqus.go @@ -44,6 +44,7 @@ type disqusComment struct { Tid uid `xml:"thread"` Pid uid `xml:"parent"` IsSpam bool `xml:"isSpam"` + Deleted bool `xml:"isDeleted"` } type uid struct { @@ -87,9 +88,10 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { commentsCh := make(chan store.Comment) stats := struct { - inpThreads, inpComments int - commentsCount, spamComments int - failedThreads, failedPosts int + inpThreads, inpComments int + commentsCount, spamComments int + failedThreads, failedPosts int + deletedComments, skippedComments int }{} go func() { @@ -101,6 +103,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { switch se := t.(type) { case xml.StartElement: + if se.Name.Local == "thread" { stats.inpThreads++ thread := disqusThread{} @@ -109,9 +112,13 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { stats.failedThreads++ continue } + if thread.Deleted { + continue + } postsMap[thread.UID] = thread.Link continue } + if se.Name.Local == "post" { stats.inpComments++ comment := disqusComment{} @@ -120,13 +127,24 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { stats.failedPosts++ continue } + if comment.Deleted { + stats.deletedComments++ + continue + } if comment.IsSpam { stats.spamComments++ continue } + + url, ok := postsMap[comment.Tid.Val] + if !ok { + stats.skippedComments++ + continue + } + c := store.Comment{ ID: comment.UID, - Locator: store.Locator{URL: postsMap[comment.Tid.Val], SiteID: siteID}, + Locator: store.Locator{URL: url, SiteID: siteID}, User: store.User{ ID: "disqus_" + store.EncodeID(comment.AuthorUserName), Name: comment.AuthorName, @@ -159,6 +177,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { } func (*Disqus) cleanText(text string) string { + text = strings.TrimSpace(text) text = strings.Replace(text, "\n", "", -1) text = strings.Replace(text, "\t", "", -1) return text diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index 8ad9c573..b3c88e6a 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -23,7 +23,9 @@ func TestDisqus_Import(t *testing.T) { dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")} defer dataStore.Close() d := Disqus{DataStore: &dataStore} - size, err := d.Import(strings.NewReader(xmlTestDisqus), "test") + fh, err := os.Open("testdata/disqus.xml") + require.NoError(t, err) + size, err := d.Import(fh, "test") assert.NoError(t, err) assert.Equal(t, 4, size) @@ -54,9 +56,71 @@ func TestDisqus_Import(t *testing.T) { assert.Equal(t, 2, count) } +func TestDisqus_ImportDeletedThread(t *testing.T) { + defer os.Remove("/tmp/remark-test.db") + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + require.NoError(t, err, "create store") + dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")} + defer dataStore.Close() + d := Disqus{DataStore: &dataStore} + fh, err := os.Open("testdata/disqus-deleted-thread.xml") + require.NoError(t, err) + size, err := d.Import(fh, "test") + assert.NoError(t, err) + assert.Equal(t, 2, size) + + last, err := dataStore.Last("test", 10, time.Time{}, adminUser) + assert.NoError(t, err) + require.Equal(t, 2, len(last), "2 comments imported") + + c := last[len(last)-1] // last reverses, get first one + assert.True(t, strings.HasPrefix(c.Text, "

Google App Engine "), c.Text) + assert.Equal(t, "299986072", c.ID) + assert.Equal(t, "", c.ParentID) + assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator) + assert.Equal(t, "No Username", c.User.Name) + assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID) + assert.Equal(t, "7001968ea3f6c9013a9f0a3650f200c10c927638", c.User.IP) + assert.True(t, c.Imported) + + c = last[1] // get comment with empty username + assert.Equal(t, "No Username", c.User.Name) + assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID) +} + +func TestDisqus_ImportDeletedPost(t *testing.T) { + defer os.Remove("/tmp/remark-test.db") + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + require.NoError(t, err, "create store") + dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")} + defer dataStore.Close() + d := Disqus{DataStore: &dataStore} + fh, err := os.Open("testdata/disqus-deleted-post.xml") + require.NoError(t, err) + size, err := d.Import(fh, "test") + assert.NoError(t, err) + assert.Equal(t, 3, size, "1 post deleted") + + last, err := dataStore.Last("test", 10, time.Time{}, adminUser) + assert.NoError(t, err) + require.Equal(t, 3, len(last), "3 comments imported") + + c := last[len(last)-1] // last reverses, get first one + assert.True(t, strings.HasPrefix(c.Text, "

Microsoft "), c.Text) + assert.Equal(t, "299744309", c.ID) + assert.Equal(t, "", c.ParentID) + assert.Equal(t, store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"}, c.Locator) + assert.Equal(t, "mikhail", c.User.Name) + assert.Equal(t, "disqus_1b6709749c0cab163db9070cc4edf3322b398d8c", c.User.ID) + assert.Equal(t, "9d3657a95a4e341510404bd8bf1a363faefd4ba4", c.User.IP) + assert.True(t, c.Imported) +} + func TestDisqus_Convert(t *testing.T) { d := Disqus{} - ch := d.convert(strings.NewReader(xmlTestDisqus), "test") + fh, err := os.Open("testdata/disqus.xml") + require.NoError(t, err) + ch := d.convert(fh, "test") res := []store.Comment{} for comment := range ch { @@ -81,155 +145,3 @@ func TestDisqus_Convert(t *testing.T) { exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z") assert.Equal(t, exp0, res[0]) } - -var xmlTestDisqus = ` - - - - radiot - General - true - - - - - radiot - - http://radio-t.umputun.com/2011/03/229_8880.html - Радио-Т: Радио-Т 229 - - 2011-03-07T20:46:25Z - - umputun@gmail.com - Umputun - false - umputun - - 98.212.28.115 - false - false - - - - http://www.radio-t.com/p/2011/03/05/podcast-229/ - radiot - - https://radio-t.com/p/2011/03/05/podcast-229/ - Радио-Т: Радио-Т 229 - - 2011-03-07T21:17:17Z - - umputun@gmail.com - Umputun - false - umputun - - 80.250.214.235 - true - false - - - - - 3565798471341011339 - - The quick brown fox jumps over the lazy dog.

some link

]]> - - 2011-08-31T15:16:29Z - false - false - - - Alexander Blah - false - facebook-1787732238 - - 178.178.178.178 - - - - - 3029154520436241933 - - Microsoft показал проводник Windows 8 с ленточным интерфейсом.

http://blogs.msdn.com/b/b8/... 

]]> - - 2011-08-31T17:44:22Z - false - false - - mihail.noname@gmail.com - mikhail - false - mikhail-noname - - 195.195.195.139 - - - - - 6580890074280459209 - - Google App Engine скоро выходит из превью статуса.

Сейчас письмо пришло от гугла.

Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.
Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).
В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.
Всем кто включит биллинг до 31 октября, обещают 50 баксов :)

]]> - - 2011-08-31T22:48:43Z - false - false - - john.nousername@gmail.com - No Username - false - - 89.89.89.139 - - - - - 12345678890 - This comment had no ID - 2011-08-31T22:49:43Z - radiot - false - false - - blah.noname@gmail.com - Blah Noname - false - 74b9e7568ef6860e93862c5d77590123 - - 189.89.89.139 - - - - - 6580890074280459219 - some ugly spam - 2011-09-30T22:48:43Z - false - true - - spam.noname@gmail.com - Spam Noname - false - google-2c5d77590123 - - 189.89.89.139 - - - - - some bad comment - 2011-x09-30T22:48:43Z - false - 123 - - noname@gmail.com - Noname - true - google-2c5d77590123 - - 189.89.89.39 - - - - -` diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 71c86d8d..9ec4fc6d 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -19,19 +19,15 @@ import ( func TestMigrator_ImportDisqus(t *testing.T) { defer func() { os.Remove("/tmp/remark-test.db") - os.Remove("/tmp/disqus-test.xml") }() - err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600) - require.NoError(t, err) - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.NoError(t, err, "create store") dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")} defer dataStore.Close() size, err := ImportComments(ImportParams{ DataStore: dataStore, - InputFile: "/tmp/disqus-test.xml", + InputFile: "testdata/disqus.xml", SiteID: "test", Provider: "disqus", }) diff --git a/backend/app/migrator/testdata/disqus-deleted-post.xml b/backend/app/migrator/testdata/disqus-deleted-post.xml new file mode 100644 index 00000000..0be4cfa3 --- /dev/null +++ b/backend/app/migrator/testdata/disqus-deleted-post.xml @@ -0,0 +1,150 @@ +` + + + + radiot + General + true + + + + + radiot + + http://radio-t.umputun.com/2011/03/229_8880.html + Радио-Т: Радио-Т 229 + + 2011-03-07T20:46:25Z + + umputun@gmail.com + Umputun + false + umputun + + 98.212.28.115 + false + false + + + + http://www.radio-t.com/p/2011/03/05/podcast-229/ + radiot + + https://radio-t.com/p/2011/03/05/podcast-229/ + Радио-Т: Радио-Т 229 + + 2011-03-07T21:17:17Z + + umputun@gmail.com + Umputun + false + umputun + + 80.250.214.235 + true + false + + + + + 3565798471341011339 + + The quick brown fox jumps over the lazy dog.

some link

]]> + + 2011-08-31T15:16:29Z + true + false + + + Alexander Blah + false + facebook-1787732238 + + 178.178.178.178 + + + + + 3029154520436241933 + + Microsoft показал проводник Windows 8 с ленточным интерфейсом.

http://blogs.msdn.com/b/b8/... 

]]> + + 2011-08-31T17:44:22Z + false + false + + mihail.noname@gmail.com + mikhail + false + mikhail-noname + + 195.195.195.139 + + + + + 6580890074280459209 + + Google App Engine скоро выходит из превью статуса.

Сейчас письмо пришло от гугла.

Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.
Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).
В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.
Всем кто включит биллинг до 31 октября, обещают 50 баксов :)

]]> + + 2011-08-31T22:48:43Z + false + false + + john.nousername@gmail.com + No Username + false + + 89.89.89.139 + + + + + 12345678890 + This comment had no ID + 2011-08-31T22:49:43Z + radiot + false + false + + blah.noname@gmail.com + Blah Noname + false + 74b9e7568ef6860e93862c5d77590123 + + 189.89.89.139 + + + + + 6580890074280459219 + some ugly spam + 2011-09-30T22:48:43Z + false + true + + spam.noname@gmail.com + Spam Noname + false + google-2c5d77590123 + + 189.89.89.139 + + + + + some bad comment + 2011-x09-30T22:48:43Z + false + 123 + + noname@gmail.com + Noname + true + google-2c5d77590123 + + 189.89.89.39 + + + + diff --git a/backend/app/migrator/testdata/disqus-deleted-thread.xml b/backend/app/migrator/testdata/disqus-deleted-thread.xml new file mode 100644 index 00000000..038e428c --- /dev/null +++ b/backend/app/migrator/testdata/disqus-deleted-thread.xml @@ -0,0 +1,150 @@ +` + + + + radiot + General + true + + + + + radiot + + http://radio-t.umputun.com/2011/03/229_8880.html + Радио-Т: Радио-Т 229 + + 2011-03-07T20:46:25Z + + umputun@gmail.com + Umputun + false + umputun + + 98.212.28.115 + false + false + + + + http://www.radio-t.com/p/2011/03/05/podcast-229/ + radiot + + https://radio-t.com/p/2011/03/05/podcast-229/ + Радио-Т: Радио-Т 229 + + 2011-03-07T21:17:17Z + + umputun@gmail.com + Umputun + false + umputun + + 80.250.214.235 + true + true + + + + + 3565798471341011339 + + The quick brown fox jumps over the lazy dog.

some link

]]> + + 2011-08-31T15:16:29Z + false + false + + + Alexander Blah + false + facebook-1787732238 + + 178.178.178.178 + + + + + 3029154520436241933 + + Microsoft показал проводник Windows 8 с ленточным интерфейсом.

http://blogs.msdn.com/b/b8/... 

]]> + + 2011-08-31T17:44:22Z + false + false + + mihail.noname@gmail.com + mikhail + false + mikhail-noname + + 195.195.195.139 + + + + + 6580890074280459209 + + Google App Engine скоро выходит из превью статуса.

Сейчас письмо пришло от гугла.

Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.
Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).
В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.
Всем кто включит биллинг до 31 октября, обещают 50 баксов :)

]]> + + 2011-08-31T22:48:43Z + false + false + + john.nousername@gmail.com + No Username + false + + 89.89.89.139 + + + + + 12345678890 + This comment had no ID + 2011-08-31T22:49:43Z + radiot + false + false + + blah.noname@gmail.com + Blah Noname + false + 74b9e7568ef6860e93862c5d77590123 + + 189.89.89.139 + + + + + 6580890074280459219 + some ugly spam + 2011-09-30T22:48:43Z + false + true + + spam.noname@gmail.com + Spam Noname + false + google-2c5d77590123 + + 189.89.89.139 + + + + + some bad comment + 2011-x09-30T22:48:43Z + false + 123 + + noname@gmail.com + Noname + true + google-2c5d77590123 + + 189.89.89.39 + + + + diff --git a/backend/app/migrator/testdata/disqus.xml b/backend/app/migrator/testdata/disqus.xml new file mode 100644 index 00000000..ff4ef6b3 --- /dev/null +++ b/backend/app/migrator/testdata/disqus.xml @@ -0,0 +1,150 @@ +` + + + + radiot + General + true + + + + + radiot + + http://radio-t.umputun.com/2011/03/229_8880.html + Радио-Т: Радио-Т 229 + + 2011-03-07T20:46:25Z + + umputun@gmail.com + Umputun + false + umputun + + 98.212.28.115 + false + false + + + + http://www.radio-t.com/p/2011/03/05/podcast-229/ + radiot + + https://radio-t.com/p/2011/03/05/podcast-229/ + Радио-Т: Радио-Т 229 + + 2011-03-07T21:17:17Z + + umputun@gmail.com + Umputun + false + umputun + + 80.250.214.235 + true + false + + + + + 3565798471341011339 + + The quick brown fox jumps over the lazy dog.

some link

]]> + + 2011-08-31T15:16:29Z + false + false + + + Alexander Blah + false + facebook-1787732238 + + 178.178.178.178 + + + + + 3029154520436241933 + + Microsoft показал проводник Windows 8 с ленточным интерфейсом.

http://blogs.msdn.com/b/b8/... 

]]> + + 2011-08-31T17:44:22Z + false + false + + mihail.noname@gmail.com + mikhail + false + mikhail-noname + + 195.195.195.139 + + + + + 6580890074280459209 + + Google App Engine скоро выходит из превью статуса.

Сейчас письмо пришло от гугла.

Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.
Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).
В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.
Всем кто включит биллинг до 31 октября, обещают 50 баксов :)

]]> + + 2011-08-31T22:48:43Z + false + false + + john.nousername@gmail.com + No Username + false + + 89.89.89.139 + + + + + 12345678890 + This comment had no ID + 2011-08-31T22:49:43Z + radiot + false + false + + blah.noname@gmail.com + Blah Noname + false + 74b9e7568ef6860e93862c5d77590123 + + 189.89.89.139 + + + + + 6580890074280459219 + some ugly spam + 2011-09-30T22:48:43Z + false + true + + spam.noname@gmail.com + Spam Noname + false + google-2c5d77590123 + + 189.89.89.139 + + + + + some bad comment + 2011-x09-30T22:48:43Z + false + 123 + + noname@gmail.com + Noname + true + google-2c5d77590123 + + 189.89.89.39 + + + +