From 1fe92d2d70d78454772bad857887adec3e11965d Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 20 Dec 2017 18:17:24 -0600 Subject: [PATCH] add rest --- app/main.go | 14 +++-- app/rest/server.go | 127 +++++++++++++++++++++++++++++++++++++---- app/store/bolt.go | 66 +++++++++++++++++---- app/store/bolt_test.go | 79 ++++++++++++++++--------- app/store/store.go | 7 ++- 5 files changed, 238 insertions(+), 55 deletions(-) diff --git a/app/main.go b/app/main.go index 77902c1e..ea985bf9 100644 --- a/app/main.go +++ b/app/main.go @@ -7,16 +7,14 @@ import ( "github.com/hashicorp/logutils" "github.com/jessevdk/go-flags" + "github.com/umputun/remark/app/store" "github.com/umputun/remark/app/rest" ) var opts struct { - Mongo []string `short:"m" long:"mongo" env:"MONGO" default:"mongo" description:"mongo host:port" env-delim:","` - MongoPasswd string `short:"p" long:"mongo-password" env:"MONGO_PASSWD" default:"" description:"mongo password"` - MongoDelay int `long:"mongo-delay" env:"MONGO_DELAY" default:"0" description:"mongo initial delay"` - - Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` + DBFile string `long:"db" env:"BOLTDB_FILE" default:"/tmp/remark.db" description:"bolt file name"` + Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` } var revision = "unknown" @@ -29,8 +27,14 @@ func main() { setupLog(opts.Dbg) log.Print("[INFO] started remark") + dataStore, err := store.NewBoltDB(opts.DBFile) + if err != nil { + log.Fatalf("[ERROR] can't initialize data store, %+v", err) + } + srv := rest.Server{ Version: revision, + Store: dataStore, } srv.Run() } diff --git a/app/rest/server.go b/app/rest/server.go index 72544bfd..e55572c8 100644 --- a/app/rest/server.go +++ b/app/rest/server.go @@ -3,16 +3,20 @@ package rest import ( "log" "net/http" + "strconv" + "strings" "time" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/go-chi/render" + "github.com/umputun/remark/app/store" ) // Server is a rest access server type Server struct { Version string + Store store.Interface } // Run the lister and request's router, activate rest server @@ -23,19 +27,122 @@ func (s *Server) Run() { router.Use(middleware.Throttle(100), middleware.Timeout(60*time.Second)) router.Use(Limiter(10), AppInfo("remark", s.Version), Ping) - router.Route("/blah", func(r chi.Router) { - r.Get("/{id}", s.getBlahCtrl) - }) - + router.Post("/comment", s.createCommentCtrl) + router.Delete("/comment/{id}", s.deleteCommentCtrl) + router.Get("/find", s.getUrlComments) + router.Get("/last/{max}", s.getLastComments) + router.Get("/id/{id}", s.getByID) log.Fatal(http.ListenAndServe(":8080", router)) } -// GET /blah/:id?foo=bar -func (s *Server) getBlahCtrl(w http.ResponseWriter, r *http.Request) { - id := chi.URLParam(r, "id") - foo := r.URL.Query().Get("foo") - log.Printf("[INFO] request for id=%s, foo=%s", id, foo) +// POST /comment +func (s *Server) createCommentCtrl(w http.ResponseWriter, r *http.Request) { + + comment := store.Comment{} + if err := render.DecodeJSON(r.Body, &comment); err != nil { + log.Printf("[WARN] can't bind request %s", comment) + httpError(w, r, http.StatusBadRequest, err, "can't bind comment") + return + } + comment.User.IP = strings.Split(r.RemoteAddr, ":")[0] + + log.Printf("[INFO] create comment %+v", comment) + + id, err := s.Store.Create(comment) + if err != nil { + log.Printf("[WARN] can't save comment, %s", err) + httpError(w, r, http.StatusInternalServerError, err, "can't save comment") + return + } render.Status(r, http.StatusAccepted) - render.JSON(w, r, JSON{"data": "something"}) + render.JSON(w, r, JSON{"id": id, "url": comment.Locator.URL}) +} + +// DELETE /comment/{id}?url=post-url +func (s *Server) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { + + id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + log.Printf("[WARN] bad id %s", chi.URLParam(r, "id")) + httpError(w, r, http.StatusBadRequest, err, "can't parse id") + } + + log.Printf("[INFO] delete comment %d", id) + + url := r.URL.Query().Get("url") + err = s.Store.Delete(url, id) + if err != nil { + log.Printf("[WARN] can't delete comment, %s", err) + httpError(w, r, http.StatusInternalServerError, err, "can't delete comment") + return + } + + render.Status(r, http.StatusOK) + render.JSON(w, r, JSON{"id": id, "url": url}) +} + +// GET /find?url=post-url +func (s *Server) getUrlComments(w http.ResponseWriter, r *http.Request) { + url := r.URL.Query().Get("url") + log.Printf("[INFO] get comments for %s", url) + + comments, err := s.Store.Find(store.Request{Locator: store.Locator{URL: url}}) + if err != nil { + log.Printf("[WARN] can't get comments for %s, %s", url, err) + httpError(w, r, http.StatusInternalServerError, err, "can't load comments comment") + return + } + render.Status(r, http.StatusOK) + render.JSON(w, r, comments) +} + +// GET /last/{max} +func (s *Server) getLastComments(w http.ResponseWriter, r *http.Request) { + + max, err := strconv.Atoi(chi.URLParam(r, "max")) + if err != nil { + max = 0 + } + + url := r.URL.Query().Get("url") + log.Printf("[INFO] get comments for %s", url) + + comments, err := s.Store.Last(store.Locator{}, max) + if err != nil { + log.Printf("[WARN] can't get last comments, %s", err) + httpError(w, r, http.StatusInternalServerError, err, "can't get last comments") + return + } + + render.Status(r, http.StatusOK) + render.JSON(w, r, comments) +} + +// GET /id/{id}?url=post-url +func (s *Server) getByID(w http.ResponseWriter, r *http.Request) { + + id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + log.Printf("[WARN] bad id %s", chi.URLParam(r, "id")) + httpError(w, r, http.StatusBadRequest, err, "can't parse id") + } + url := r.URL.Query().Get("url") + + log.Printf("[INFO] get comments by id %d, %s", id, url) + + comment, err := s.Store.Get(store.Locator{URL: url}, id) + if err != nil { + log.Printf("[WARN] can't get comment, %s", err) + httpError(w, r, http.StatusInternalServerError, err, "can't get comment by id") + return + } + + render.Status(r, http.StatusOK) + render.JSON(w, r, comment) +} + +func httpError(w http.ResponseWriter, r *http.Request, code int, err error, details string) { + render.Status(r, code) + render.JSON(w, r, JSON{"error": err.Error(), "details": details}) } diff --git a/app/store/bolt.go b/app/store/bolt.go index 8bdeddf1..1d07c220 100644 --- a/app/store/bolt.go +++ b/app/store/bolt.go @@ -35,11 +35,12 @@ func NewBoltDB(dbFile string) (*BoltDB, error) { } // Create saves new comment to store -func (b *BoltDB) Create(comment Comment) error { +func (b *BoltDB) Create(comment Comment) (int64, error) { comment.ID = time.Now().UnixNano() + comment.Timestamp = time.Now() - return b.Update(func(tx *bolt.Tx) error { + err := b.Update(func(tx *bolt.Tx) error { bucket, e := tx.CreateBucketIfNotExists([]byte(comment.Locator.URL)) if e != nil { return errors.Wrapf(e, "can't make bucket", comment.Locator.URL) @@ -75,6 +76,8 @@ func (b *BoltDB) Create(comment Comment) error { return nil }) + + return comment.ID, err } // Delete removed comment by url and id from the store @@ -85,7 +88,7 @@ func (b *BoltDB) Delete(url string, id int64) error { if bucket == nil { return errors.Errorf("no bucket %s in store", url) } - key := []byte(fmt.Sprintf("%12d", id)) + key := b.keyFromValue(id) if err := bucket.Delete(key); err != nil { return errors.Wrapf(err, "can't delete key %s from bucket %s", key, url) } @@ -116,26 +119,64 @@ func (b *BoltDB) Find(request Request) ([]Comment, error) { return res, err } +// Get comment by id +func (b *BoltDB) Get(locator Locator, id int64) (comment Comment, err error) { + err = b.View(func(tx *bolt.Tx) error { + + lastBucket := tx.Bucket([]byte(lastBucketName)) + if lastBucket == nil { + return errors.Errorf("no bucket %s in store", lastBucketName) + } + + c := lastBucket.Cursor() + for k, v := c.Last(); k != nil; k, v = c.Prev() { + url, foundID, e := refFromValue(v).parse() + if e != nil { + return e + } + + if foundID == id && url == locator.URL { + urlBucket := tx.Bucket([]byte(url)) + if urlBucket == nil { + return errors.Errorf("no bucket %s in store", url) + } + commentVal := urlBucket.Get(b.keyFromValue(id)) + if commentVal == nil { + return errors.Errorf("no comment for %d in store %s", id, url) + } + + if e := json.Unmarshal(commentVal, &comment); e != nil { + return errors.Wrap(e, "failed to unmarshal") + } + return nil + } + } + return errors.Errorf("no id %d in store %s", id, locator.URL) + }) + + return comment, err +} + // Last returns up to max last comments for given locator func (b *BoltDB) Last(locator Locator, max int) (result []Comment, err error) { err = b.View(func(tx *bolt.Tx) error { - lastBk := tx.Bucket([]byte(lastBucketName)) - if lastBk == nil { + lastBucket := tx.Bucket([]byte(lastBucketName)) + if lastBucket == nil { return errors.Errorf("no bucket %s in store", lastBucketName) } - c := lastBk.Cursor() + c := lastBucket.Cursor() for k, v := c.Last(); k != nil; k, v = c.Prev() { url, id, e := refFromValue(v).parse() if e != nil { return e } - urlBk := tx.Bucket([]byte(url)) - if urlBk == nil { + urlBucket := tx.Bucket([]byte(url)) + if urlBucket == nil { return errors.Errorf("no bucket %s in store", url) } - commentVal := urlBk.Get(b.keyFromValue(id)) + commentVal := urlBucket.Get(b.keyFromValue(id)) if commentVal == nil { return errors.Errorf("no comment for %d in store %s", id, url) } @@ -145,6 +186,9 @@ func (b *BoltDB) Last(locator Locator, max int) (result []Comment, err error) { return errors.Wrap(e, "failed to unmarshal") } result = append(result, comment) + if max > 0 && len(result) >= max { + return nil + } } return nil }) @@ -153,11 +197,11 @@ func (b *BoltDB) Last(locator Locator, max int) (result []Comment, err error) { } func (b *BoltDB) keyFromComment(comment Comment) []byte { - return []byte(fmt.Sprintf("%12d", comment.ID)) + return []byte(fmt.Sprintf("%22d", comment.ID)) } func (b *BoltDB) keyFromValue(id int64) []byte { - return []byte(fmt.Sprintf("%12d", id)) + return []byte(fmt.Sprintf("%22d", id)) } // buckets returns list of buckets, which is list of all commented posts diff --git a/app/store/bolt_test.go b/app/store/bolt_test.go index 1cedc318..47b58edb 100644 --- a/app/store/bolt_test.go +++ b/app/store/bolt_test.go @@ -11,43 +11,21 @@ import ( var testDb = "/tmp/test-remark.db" func TestBoltDB_CreateAndFind(t *testing.T) { + var b Interface defer os.Remove(testDb) - - b, err := NewBoltDB(testDb) - assert.Nil(t, err) - - comment := Comment{Text: "some text", Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}} - err = b.Create(comment) - assert.Nil(t, err) - - comment = Comment{Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}} - err = b.Create(comment) - assert.Nil(t, err) + b = prep(t) res, err := b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}}) assert.Nil(t, err) assert.Equal(t, 2, len(res)) assert.Equal(t, "some text", res[0].Text) assert.Equal(t, "user1", res[0].User.ID) + t.Log(res[0].ID) } func TestBoltDB_Delete(t *testing.T) { defer os.Remove(testDb) - - b, err := NewBoltDB(testDb) - assert.Nil(t, err) - - comment := Comment{Text: "some text", Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}} - err = b.Create(comment) - assert.Nil(t, err) - - comment = Comment{Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}} - err = b.Create(comment) - assert.Nil(t, err) + b := prep(t) res, err := b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}}) assert.Nil(t, err) @@ -61,3 +39,52 @@ func TestBoltDB_Delete(t *testing.T) { assert.Equal(t, 1, len(res)) assert.Equal(t, "some text2", res[0].Text) } + +func TestBoltDB_Get(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + res, err := b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}}) + assert.Nil(t, err) + assert.Equal(t, 2, len(res)) + + comment, err := b.Get(Locator{URL: "https://radio-t.com"}, res[1].ID) + assert.Nil(t, err) + assert.Equal(t, "some text2", comment.Text) + + comment, err = b.Get(Locator{URL: "https://radio-t.com"}, 1234567) + assert.NotNil(t, err) +} + +func TestBoltDB_Last(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + res, err := b.Last(Locator{URL: "https://radio-t.com"}, 0) + assert.Nil(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "some text2", res[0].Text) + + res, err = b.Last(Locator{URL: "https://radio-t.com"}, 1) + assert.Nil(t, err) + assert.Equal(t, 1, len(res)) + assert.Equal(t, "some text2", res[0].Text) +} + +// makes new boltdb, put two records +func prep(t *testing.T) *BoltDB { + b, err := NewBoltDB(testDb) + assert.Nil(t, err) + + comment := Comment{Text: "some text", Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}} + _, err = b.Create(comment) + assert.Nil(t, err) + + comment = Comment{Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), + Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}} + _, err = b.Create(comment) + assert.Nil(t, err) + + return b +} diff --git a/app/store/store.go b/app/store/store.go index 419f9353..4f21fa56 100644 --- a/app/store/store.go +++ b/app/store/store.go @@ -39,8 +39,9 @@ type Request struct { // Interface defines basic CRUD for comments type Interface interface { - Create(comment Comment) error - Delete(id string) error + Create(comment Comment) (int64, error) + Delete(url string, id int64) error Find(request Request) ([]Comment, error) - Last(locator Locator, max int) []Comment + Last(locator Locator, max int) ([]Comment, error) + Get(locator Locator, id int64) (Comment, error) }