mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Eugene
2017-12-20 15:29:54 -06:00
parent 299d3db2e6
commit eb32dda24c
315 changed files with 140636 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.go
+52
View File
@@ -0,0 +1,52 @@
package main
import (
"fmt"
"log"
"os"
"github.com/hashicorp/logutils"
"github.com/jessevdk/go-flags"
"github.com/umputun/remark/app/rest"
)
var opts struct {
Mongo []string `short:"m" long:"mongo" env:"MONGO" default:"mongo" description:"mongo host:port" env-delim:","`
MongoPasswd string `short:"p" long:"mongo-password" env:"MONGO_PASSWD" default:"" description:"mongo password"`
MongoDelay int `long:"mongo-delay" env:"MONGO_DELAY" default:"0" description:"mongo initial delay"`
Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
}
var revision = "unknown"
func main() {
fmt.Printf("remark %s\n", revision)
if _, err := flags.Parse(&opts); err != nil {
log.Fatal(err)
}
setupLog(opts.Dbg)
log.Print("[INFO] started remark")
srv := rest.Server{
Version: revision,
}
srv.Run()
}
func setupLog(dbg bool) {
filter := &logutils.LevelFilter{
Levels: []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR"},
MinLevel: logutils.LogLevel("INFO"),
Writer: os.Stdout,
}
log.SetFlags(log.Ldate | log.Ltime)
if dbg {
log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
filter.MinLevel = logutils.LogLevel("DEBUG")
}
log.SetOutput(filter)
}
app/rest/middleware.go
+104
View File
@@ -0,0 +1,104 @@
package rest
import (
"log"
"net/http"
"os"
"strings"
"time"
"github.com/didip/tollbooth"
"github.com/go-chi/render"
)
var org = "Umputun"
// JSON is a map alias, just for convenience
type JSON map[string]interface{}
// Limiter middleware defines max recs/sec for given client. Client detected as a combination
// of source IP, auth key and user agent. Requests rejected with 429 status code.
func Limiter(recSec int, excludeIps ...string) func(http.Handler) http.Handler {
return func(h http.Handler) http.Handler {
l := tollbooth.NewLimiter(int64(recSec), time.Second)
fn := func(w http.ResponseWriter, r *http.Request) {
for _, exclIP := range excludeIps {
if strings.HasPrefix(r.RemoteAddr, exclIP) {
h.ServeHTTP(w, r)
return
}
}
keys := []string{
r.Header.Get("Authorization"),
r.Header.Get("X-Forwarded-For"),
r.Header.Get("X-Real-IP"),
r.Header.Get("RemoteAddr"),
r.Header.Get("User-Agent"),
}
if httpError := tollbooth.LimitByKeys(l, keys); httpError != nil {
render.Status(r, httpError.StatusCode)
render.JSON(w, r, JSON{"error": httpError.Message})
return
}
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
// AppInfo adds custom app-info to header
func AppInfo(app string, version string) func(http.Handler) http.Handler {
f := func(h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Org", org)
w.Header().Set("App-Name", app)
w.Header().Set("App-Version", version)
if mhost := os.Getenv("MHOST"); mhost != "" {
w.Header().Set("Host", mhost)
}
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
return f
}
// Ping middleware response with pong. Stops chain if ping request detected
func Ping(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/ping") {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
if _, err := w.Write([]byte("pong")); err != nil {
log.Printf("[WARN] can't send pong, %s", err)
}
return
}
next.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
// Recoverer is a middleware that recovers from panics, logs the panic and returns a HTTP 500 status if possible.
func Recoverer(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
defer func() {
if rvr := recover(); rvr != nil {
log.Printf("[ERROR] request panic, %v", rvr)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}
}()
next.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
app/rest/server.go
+41
View File
@@ -0,0 +1,41 @@
package rest
import (
"log"
"net/http"
"time"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
"github.com/go-chi/render"
)
// Server is a rest access server
type Server struct {
Version string
}
// Run the lister and request's router, activate rest server
func (s *Server) Run() {
log.Print("[INFO] activate rest server")
router := chi.NewRouter()
router.Use(middleware.RealIP, Recoverer)
router.Use(middleware.Throttle(100), middleware.Timeout(60*time.Second))
router.Use(Limiter(10), AppInfo("remark", s.Version), Ping)
router.Route("/blah", func(r chi.Router) {
r.Get("/{id}", s.getBlahCtrl)
})
log.Fatal(http.ListenAndServe(":8080", router))
}
// GET /blah/:id?foo=bar
func (s *Server) getBlahCtrl(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
foo := r.URL.Query().Get("foo")
log.Printf("[INFO] request for id=%s, foo=%s", id, foo)
render.Status(r, http.StatusAccepted)
render.JSON(w, r, JSON{"data": "something"})
}
app/store/bolt.go
+199
View File
@@ -0,0 +1,199 @@
package store
import (
"encoding/json"
"fmt"
"log"
"strconv"
"strings"
"time"
"github.com/boltdb/bolt"
"github.com/pkg/errors"
)
// BoltDB implements store.Interface. Each instance represents one site.
// Keys built as pid-id. Each url (post) makes it's own bucket
// In addition there is a bucket "last" with reference to other buckets+keys to all cross-posts last comment extraction.
// Thread safe.
type BoltDB struct {
*bolt.DB
}
var lastBucketName = "last"
// NewBoltDB makes persistent boltdb-based store
func NewBoltDB(dbFile string) (*BoltDB, error) {
log.Printf("[INFO] bolt store, %s", dbFile)
result := BoltDB{}
db, err := bolt.Open(dbFile, 0600, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return nil, err
}
result.DB = db
return &result, err
}
// Create saves new comment to store
func (b *BoltDB) Create(comment Comment) error {
comment.ID = time.Now().UnixNano()
return b.Update(func(tx *bolt.Tx) error {
bucket, e := tx.CreateBucketIfNotExists([]byte(comment.Locator.URL))
if e != nil {
return errors.Wrapf(e, "can't make bucket", comment.Locator.URL)
}
// check if key already in store, reject doubles
key := b.keyFromComment(comment)
if bucket.Get(key) != nil {
return errors.Errorf("key %s already in store", string(key))
}
// serialise comment to json's []byte for bolt and save
jdata, jerr := json.Marshal(&comment)
if jerr != nil {
return errors.Wrap(jerr, "can't marshal comment")
}
if err := bucket.Put(key, jdata); err != nil {
return errors.Wrapf(err, "failed to put key %s", string(key))
}
// add reference to comment to "last" bucket
bucket, e = tx.CreateBucketIfNotExists([]byte(lastBucketName))
if e != nil {
return errors.Wrapf(e, "can't make bucket %s", lastBucketName)
}
rv := refFromComment(comment)
e = bucket.Put([]byte(fmt.Sprintf("%d", time.Now().UnixNano())), []byte(rv.value()))
if e != nil {
return errors.Wrapf(e, "can't put reference %s to %s", rv.value(), lastBucketName)
}
return nil
})
}
// Delete removed comment by url and id from the store
func (b *BoltDB) Delete(url string, id int64) error {
return b.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(url))
if bucket == nil {
return errors.Errorf("no bucket %s in store", url)
}
key := []byte(fmt.Sprintf("%12d", id))
if err := bucket.Delete(key); err != nil {
errors.Wrapf(err, "can't delete key %s from bucket %s", key, url)
}
return nil
})
}
// Find comments for post
func (b *BoltDB) Find(request Request) ([]Comment, error) {
res := []Comment{}
err := b.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(request.Locator.URL))
if bucket == nil {
return errors.Errorf("no bucket %s in store", request.Locator.URL)
}
return bucket.ForEach(func(k, v []byte) error {
comment := Comment{}
if e := json.Unmarshal(v, &comment); e != nil {
return errors.Wrap(e, "failed to unmarshal")
}
res = append(res, comment)
return nil
})
})
return res, err
}
// Last returns up to max last comments for given locator
func (b *BoltDB) Last(locator Locator, max int) (result []Comment, err error) {
err = b.View(func(tx *bolt.Tx) error {
lastBk := tx.Bucket([]byte(lastBucketName))
if lastBk == nil {
return errors.Errorf("no bucket %s in store", lastBucketName)
}
c := lastBk.Cursor()
for k, v := c.Last(); k != nil; k, v = c.Prev() {
url, id, err := refFromValue(v).parse()
if err != nil {
return err
}
urlBk := tx.Bucket([]byte(url))
if urlBk == nil {
return errors.Errorf("no bucket %s in store", url)
}
commentVal := urlBk.Get(b.keyFromValue(id))
if commentVal == nil {
return errors.Errorf("no comment for %d in store %s", id, url)
}
comment := Comment{}
if e := json.Unmarshal(commentVal, &comment); e != nil {
return errors.Wrap(e, "failed to unmarshal")
}
result = append(result, comment)
}
return nil
})
return result, err
}
func (b *BoltDB) keyFromComment(comment Comment) []byte {
return []byte(fmt.Sprintf("%12d", comment.ID))
}
func (b *BoltDB) keyFromValue(id int64) []byte {
return []byte(fmt.Sprintf("%12d", id))
}
// buckets returns list of buckets, which is list of all commented posts
func (b BoltDB) buckets() (result []string) {
b.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
result = append(result, string(name))
return nil
})
})
return result
}
type ref string
func refFromComment(comment Comment) *ref {
result := ref(fmt.Sprintf("%s!!%d", comment.Locator.URL, comment.ID))
return &result
}
func refFromValue(val []byte) *ref {
result := ref(string(val))
return &result
}
func (r ref) value() string { return string(r) }
func (r ref) parse() (url string, id int64, err error) {
elems := strings.Split(string(r), "!!")
if len(elems) < 2 {
return "", 0, errors.Errorf("can't parse ref %s", r)
}
url = elems[0]
if id, err = strconv.ParseInt(elems[1], 10, 64); err != nil {
return "", 0, errors.Wrapf(err, "can't extract id from ref %s", r)
}
return url, id, nil
}
app/store/bolt_test.go
+63
View File
@@ -0,0 +1,63 @@
package store
import (
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var testDb = "/tmp/test-remark.db"
func TestBoltDB_CreateAndFind(t *testing.T) {
defer os.Remove(testDb)
b, err := NewBoltDB(testDb)
assert.Nil(t, err)
comment := Comment{Text: "some text", Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}}
err = b.Create(comment)
assert.Nil(t, err)
comment = Comment{Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}}
err = b.Create(comment)
assert.Nil(t, err)
res, err := b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}})
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text", res[0].Text)
assert.Equal(t, "user1", res[0].User.ID)
}
func TestBoltDB_Delete(t *testing.T) {
defer os.Remove(testDb)
b, err := NewBoltDB(testDb)
assert.Nil(t, err)
comment := Comment{Text: "some text", Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}}
err = b.Create(comment)
assert.Nil(t, err)
comment = Comment{Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
Locator: Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}}
err = b.Create(comment)
assert.Nil(t, err)
res, err := b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}})
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
err = b.Delete("https://radio-t.com", res[0].ID)
assert.Nil(t, err)
res, err = b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}})
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
}
app/store/store.go
+46
View File
@@ -0,0 +1,46 @@
package store
import "time"
// Comment represents a single comment with reference to its parent
type Comment struct {
ID int64 `json:"id"`
ParentID int64 `json:"pid"`
Text string `json:"text"`
User User `json:"user"`
Locator Locator `json:"locator"`
Score int `json:"score"`
Timestamp time.Time `json:"time"`
}
// Locator keeps site and url of the post
type Locator struct {
SiteID string `json:"site"`
URL string `json:"url"`
}
// User holds user-related info
type User struct {
Name string `json:"name"`
ID string `json:"id"`
IP string `json:"-"`
}
// Request is a container for all finds
type Request struct {
Locator Locator `json:"locator"`
Sort string `json:"sort"`
Offset int `json:"offset"`
Limit int `json:"limit"`
}
// Interface defines basic CRUD for comments
type Interface interface {
Create(comment Comment) error
Delete(id string) error
Find(request Request) ([]Comment, error)
Last(locator Locator, max int) []Comment
}