sync with preemptuve lock in concurrent migrator

This commit is contained in:
Umputun
2019-02-02 18:45:24 -06:00
parent 588af7fc9a
commit e6f30b0768
3 changed files with 8 additions and 5 deletions
+5 -4
View File
@@ -109,7 +109,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
if n.Concurrent > 0 {
concurrent = n.Concurrent
}
grp := syncs.NewSizedGroup(concurrent)
grp := syncs.NewErrSizedGroup(concurrent, syncs.Preemptive())
for {
comment := store.Comment{}
@@ -127,21 +127,22 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
}
// write comments in parallel
grp.Go(func() {
grp.Go(func() error {
if _, e := n.DataStore.Create(comment); e != nil {
atomic.AddInt64(&failed, 1)
log.Printf("[WARN] can't write %+v to store, %s", comment, e)
return
return nil
}
n := atomic.AddInt64(&comments, 1)
if n%1000 == 0 {
log.Printf("[DEBUG] imported %d comments", n)
}
return nil
})
}
grp.Wait()
_ = grp.Wait()
if failed > 0 {
return int(comments), errors.Errorf("failed to save %d comments", failed)
+1 -1
View File
@@ -75,7 +75,7 @@ func TestAdmin_Delete(t *testing.T) {
assert.Equal(t, "", cr.Text)
assert.True(t, cr.Deleted)
time.Sleep(100 * time.Millisecond)
time.Sleep(250 * time.Millisecond)
// check last comments updated
res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
assert.Equal(t, 200, code)
@@ -145,6 +145,8 @@ func TestService_SetTitle(t *testing.T) {
t.Logf("%+v", res)
assert.Equal(t, "", res.PostTitle)
b.TitleExtractor.cache.Purge()
atomic.StoreInt32(&titleEnable, 1)
c, err := b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
require.NoError(t, err)