mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Imported field to comment, do not wait image submission for such comments

Browse Source
This commit is contained in:
vdimir
2020-07-09 12:43:07 -05:00
committed by Umputun
parent 1f9566bb47
commit 9957ab642b
8 changed files with 44 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7 -2
View File
@@ -292,9 +292,9 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
1. Register a new application [using the Azure portal](https://docs.microsoft.com/en-us/graph/auth-register-app-v2).
2. Under **"Authentication/Platform configurations/Web"** enter the correct url constructed as domain + `/auth/microsoft/callback`. i.e. `https://example.mysite.com/auth/microsoft/callback`
3. In "Overview" take note of the **Application (client) ID**
3. In "Overview" take note of the **Application (client) ID**
4. Choose the new project from the top right project dropdown (only if another project is selected)
5. Select "Certificates & secrets" and click on "+ New Client Secret".
5. Select "Certificates & secrets" and click on "+ New Client Secret".
##### Twitter Auth Provider
@@ -324,6 +324,11 @@ Optionally, anonymous access can be turned on. In this case an extra `anonymous`
- name should be at least 3 characters long
- name has to start from the letter and contains letters, numbers, underscores and spaces only.
### Importing comments
Remark supports importing comments from Disqus, WordPress or native backup format.
All imported comments has `Imported` field set to `true`.
#### Initial import from Disqus
1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
backend/app/migrator/disqus.go
+1
View File
@@ -135,6 +135,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
Text: d.cleanText(comment.Message),
Timestamp: comment.CreatedAt,
ParentID: comment.Pid.Val,
Imported: true,
}
if c.User.ID == "disqus_" { // empty comment.AuthorUserName from disqus
c.User.ID = "disqus_" + c.User.Name
backend/app/migrator/native.go
+1
View File
@@ -155,6 +155,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
for {
comment := store.Comment{}
err = dec.Decode(&comment)
comment.Imported = true
if err == io.EOF {
break
}
backend/app/migrator/wordpress.go
+1
View File
@@ -139,6 +139,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
Text: comment.Content,
Timestamp: comment.Date.time,
ParentID: comment.PID,
Imported: true,
}
commentsCh <- commentFormatter.Format(c)
stats.inpComments++
backend/app/store/comment.go
+1
View File
@@ -26,6 +26,7 @@ type Comment struct {
Edit *Edit `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
Pin bool `json:"pin,omitempty" bson:"pin,omitempty"`
Deleted bool `json:"delete,omitempty" bson:"delete"`
Imported bool `json:"imported,omitempty" bson:"imported"`
PostTitle string `json:"title,omitempty" bson:"title"`
}
backend/app/store/image/image.go
+12 -11
View File
@@ -101,8 +101,17 @@ func NewService(s Store, p ServiceParams) *Service {
return &Service{ServiceParams: p, store: s}
}
// SubmitAndCommit multiple ids immediately
func (s *Service) SubmitAndCommit(idsFn func() []string) {
for _, id := range idsFn() {
if err := s.store.Commit(id); err != nil {
log.Printf("[WARN] failed to commit image %s", id)
}
}
}
// Submit multiple ids via function for delayed commit
func (s *Service) Submit(idsFn func() []string, ts time.Time) {
func (s *Service) Submit(idsFn func() []string) {
if idsFn == nil || s == nil {
return
}
@@ -118,11 +127,7 @@ func (s *Service) Submit(idsFn func() []string, ts time.Time) {
for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.commitTTL {
time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
}
for _, id := range req.idsFn() {
if err := s.store.Commit(id); err != nil {
log.Printf("[WARN] failed to commit image %s", id)
}
}
s.SubmitAndCommit(req.idsFn)
atomic.AddInt32(&s.submitCount, -1)
}
log.Printf("[INFO] image submitter terminated")
@@ -131,11 +136,7 @@ func (s *Service) Submit(idsFn func() []string, ts time.Time) {
atomic.AddInt32(&s.submitCount, 1)
now := time.Now()
if ts.IsZero() || ts.After(now) {
ts = now
}
s.submitCh <- submitReq{idsFn: idsFn, TS: ts}
s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
}
// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
backend/app/store/image/image_test.go
+13 -12
View File
@@ -135,14 +135,15 @@ func TestService_Cleanup(t *testing.T) {
func TestService_Submit(t *testing.T) {
store := MockStore{}
store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
store.On("Commit", mock.Anything, mock.Anything).Times(7).Return(nil)
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}}
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now())
svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now())
svc.Submit(nil, time.Now())
store.AssertNumberOfCalls(t, "Commit", 0)
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} })
svc.Submit(func() []string { return []string{"id6", "id7"} })
svc.Submit(nil)
store.AssertNumberOfCalls(t, "Commit", 2)
time.Sleep(time.Millisecond * 150)
store.AssertNumberOfCalls(t, "Commit", 5)
store.AssertNumberOfCalls(t, "Commit", 7)
svc.Close(context.TODO())
}
@@ -150,9 +151,9 @@ func TestService_Close(t *testing.T) {
store := MockStore{}
store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}}
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now())
svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now())
svc.Submit(nil, time.Now())
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
svc.Submit(func() []string { return []string{"id4", "id5"} })
svc.Submit(nil)
svc.Close(context.TODO())
store.AssertNumberOfCalls(t, "Commit", 5)
}
@@ -161,10 +162,10 @@ func TestService_SubmitDelay(t *testing.T) {
store := MockStore{}
store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now())
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now())
svc.Submit(nil, time.Now())
svc.Submit(func() []string { return []string{"id4", "id5"} })
svc.Submit(nil)
store.AssertNumberOfCalls(t, "Commit", 3)
svc.Close(context.TODO())
store.AssertNumberOfCalls(t, "Commit", 5)
backend/app/store/service/service.go
+8 -3
View File
@@ -225,8 +225,7 @@ func (s *DataStore) ResubmitStagingImages(sites []string) error {
// submitImages initiated delayed commit of all images from the comment uploaded to remark42
func (s *DataStore) submitImages(comment store.Comment) {
s.ImageService.Submit(func() []string { // get all ids from comment's text
idsFn := func() []string { // get all ids from comment's text
// this can be called after last edit, we have to retrieve fresh comment
cc, err := s.Engine.Get(engine.GetRequest{Locator: comment.Locator, CommentID: comment.ID})
if err != nil {
@@ -242,7 +241,13 @@ func (s *DataStore) submitImages(comment store.Comment) {
log.Printf("[DEBUG] image ids extracted from %s - %+v", comment.ID, imgIds)
}
return imgIds
}, comment.Timestamp)
}
if comment.Imported {
s.ImageService.SubmitAndCommit(idsFn)
} else {
s.ImageService.Submit(idsFn)
}
}
// prepareNewComment sets new comment fields, hashing and sanitizing data