diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go index 9f3fba73..2d80104f 100644 --- a/app/migrator/disqus.go +++ b/app/migrator/disqus.go @@ -7,8 +7,6 @@ import ( "log" "time" - "sync" - "strings" "github.com/pkg/errors" @@ -18,9 +16,6 @@ import ( // Disqus implements Importer from disqus xml type Disqus struct { DataStore store.Interface - - ch chan store.Comment - once sync.Once } type disqusXML struct { @@ -48,6 +43,7 @@ type disqusComment struct { ID string `xml:"id"` Message string `xml:"message"` CreatedAt time.Time `xml:"createdAt"` + IsSpam bool `xml:"isSpam"` AuthorEmail string `xml:"author>email"` AuthorName string `xml:"author>name"` AuthorUserName string `xml:"author>username"` @@ -114,6 +110,9 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { if err := decoder.DecodeElement(&comment, &se); err != nil { continue } + if comment.IsSpam { + continue + } c := store.Comment{ ID: comment.ID, Locator: store.Locator{URL: postsMap[comment.Tid.Val], SiteID: siteID}, diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go index bc8b7438..3a4b00ab 100644 --- a/app/migrator/disqus_test.go +++ b/app/migrator/disqus_test.go @@ -19,7 +19,7 @@ func TestDisqus_Convert(t *testing.T) { res = append(res, comment) t.Logf("%+v", comment) } - assert.Equal(t, 3, len(res), "3 comments total") + assert.Equal(t, 3, len(res), "3 comments total, 1 spam excluded") exp0 := store.Comment{ ID: "3565798471341011339", @@ -29,9 +29,9 @@ func TestDisqus_Convert(t *testing.T) { }, Text: "

The quick brown fox jumps over the lazy dog.

", User: store.User{ - Name: "Alexander Puzatykh", + Name: "Alexander Blah", ID: "facebook-1787732238", - IP: "178.234.205.125", + IP: "178.178.178.178", }, } exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z") @@ -89,11 +89,11 @@ var xmlTest = ` false - Alexander Puzatykh + Alexander Blah false facebook-1787732238 - 178.234.205.125 + 178.178.178.178 @@ -106,12 +106,12 @@ var xmlTest = ` false false - mihail.merkulov@gmail.com - mikhailmerkulov + mihail.noname@gmail.com + mikhail false - mikhailmerkulov + mikhail-noname - 195.234.75.139 + 195.195.195.139 @@ -124,12 +124,30 @@ var xmlTest = ` false false - unikier@gmail.com - Dmitry Shapoval + dmitri.noname@gmail.com + Dmitry Noname false - google-74b9e7568ef6860e93862c5d7752b657 + google-74b9e7568ef6860e93862c5d77590123 - 89.113.25.139 + 89.89.89.139 + + + + + 6580890074280459219 + + some ugly spam + + 2011-09-31T22:48:43Z + false + true + + spam.noname@gmail.com + Spam Noname + false + google-2c5d77590123 + + 189.89.89.139 diff --git a/app/migrator/importer.go b/app/migrator/migrator.go similarity index 54% rename from app/migrator/importer.go rename to app/migrator/migrator.go index d33bdc62..31d788a0 100644 --- a/app/migrator/importer.go +++ b/app/migrator/migrator.go @@ -6,3 +6,8 @@ import "io" type Importer interface { Import(r io.Reader, siteID string) error } + +// Exporter defines interface to export comments from internal store +type Exporter interface { + Export(w io.Writer, siteID string) error +}