From be46e849a4836917053ccfa3dab5ecf15dae8dd7 Mon Sep 17 00:00:00 2001 From: bakurin Date: Sun, 8 Aug 2021 21:58:03 +0300 Subject: [PATCH] Webhook destination for notifications --- README.md | 10 +- backend/app/cmd/server.go | 70 ++++++++++++- backend/app/cmd/server_test.go | 23 +++++ backend/app/main_test.go | 63 ++++++++++++ backend/app/notify/webhook.go | 113 ++++++++++++++++++++ backend/app/notify/webhook_test.go | 160 +++++++++++++++++++++++++++++ 6 files changed, 435 insertions(+), 4 deletions(-) create mode 100644 backend/app/notify/webhook.go create mode 100644 backend/app/notify/webhook_test.go diff --git a/README.md b/README.md index 9626482f..b9827f29 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Remark42 is a self-hosted, lightweight and simple (yet functional) comment engin * Images upload with drag-and-drop * Extractor for recent comments, cross-post * RSS for all comments and each post -* Telegram, Slack and email notifications for Admins (get notified for each new comment) +* Telegram, Slack, Webhook and email notifications for Admins (get notified for each new comment) * Email and Telegram notifications for users (get notified when someone responds to your comment) * Export data to JSON with automatic backups * No external databases, everything embedded in a single data file @@ -161,12 +161,16 @@ _this is the recommended way to run Remark42_ | auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html` | email content type | | auth.email.template | AUTH_EMAIL_TEMPLATE | none (predefined) | custom email message template file | | notify.users | NOTIFY_USERS | none | type of user notifications (Telegram, email) | -| notify.admins | NOTIFY_ADMINS | none | type of admin notifications (Telegram, Slack and/or email) | +| notify.admins | NOTIFY_ADMINS | none | type of admin notifications (Telegram, Slack, webhook and/or email) | | notify.queue | NOTIFY_QUEUE | `100` | size of notification queue | | notify.telegram.chan | NOTIFY_TELEGRAM_CHAN | | Telegram channel | | notify.slack.token | NOTIFY_SLACK_TOKEN | | Slack token | | notify.slack.chan | NOTIFY_SLACK_CHAN | `general` | Slack channel | -| notify.email.fromAddress | NOTIFY_EMAIL_FROM | | from email address | +| notify.webhook.url | NOTIFY_WEBHOOK_URL | | Webhook notification URL | +| notify.webhook.template | NOTIFY_WEBHOOK_TEMPLATE | `{"text": "{{.Text}}"}` | Webhook payload template | +| notify.webhook.headers | NOTIFY_WEBHOOK_HEADERS | | HTTP header in format Header1:Value1,Header2:Value2,...| +| notify.webhook.timeout | NOTIFY_WEBHOOK_TIMEOUT | `5s` | Webhook connection timeout | +| notify.email.fromAddress| NOTIFY_EMAIL_FROM | | from email address | | notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject | | telegram.token | TELEGRAM_TOKEN | | Telegram token (used for auth and Telegram notifications) | | telegram.timeout | TELEGRAM_TIMEOUT | `5s` | Telegram connection timeout | diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 8c0d4e31..723459ed 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -213,7 +213,7 @@ type SMTPGroup struct { type NotifyGroup struct { Type []string `long:"type" env:"TYPE" description:"[deprecated, use user and admin types instead] types of notifications" choice:"none" choice:"telegram" choice:"email" choice:"slack" default:"none" env-delim:","` //nolint Users []string `long:"users" env:"USERS" description:"types of user notifications" choice:"none" choice:"email" choice:"telegram" default:"none" env-delim:","` //nolint - Admins []string `long:"admins" env:"ADMINS" description:"types of admin notifications" choice:"none" choice:"telegram" choice:"email" choice:"slack" default:"none" env-delim:","` //nolint + Admins []string `long:"admins" env:"ADMINS" description:"types of admin notifications" choice:"none" choice:"telegram" choice:"email" choice:"slack" choice:"webhook" default:"none" env-delim:","` //nolint QueueSize int `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"` Telegram struct { Channel string `long:"chan" env:"CHAN" description:"telegram channel for admin notifications"` @@ -230,6 +230,12 @@ type NotifyGroup struct { Token string `long:"token" env:"TOKEN" description:"slack token"` Channel string `long:"chan" env:"CHAN" description:"slack channel"` } `group:"slack" namespace:"slack" env-namespace:"SLACK"` + Webhook struct { + WebhookURL string `long:"url" env:"URL" description:"webhook notification URL"` + Template string `long:"template" env:"TEMPLATE" description:"webhook authentication template" default:"{\"text\": \"{{.Text}}\"}"` + Headers []string `long:"headers" description:"webhook authentication headers in format --notify.webhook.headers=Header1:Value1,Value2,..."` // env NOTIFY_WEBHOOK_HEADERS split in code bellow to allow , inside "" + Timeout time.Duration `long:"timeout" env:"TIMEOUT" description:"webhook timeout" default:"5s"` + } `group:"webhook" namespace:"webhook" env-namespace:"WEBHOOK"` } // SSLGroup defines options group for server ssl params @@ -903,6 +909,25 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator * var destinations []notify.Destination var telegramBotUsername string + if contains("webhook", s.Notify.Admins) { + client := &http.Client{Timeout: 5 * time.Second} + webhookHeaders := s.Notify.Webhook.Headers + if len(webhookHeaders) == 0 { + webhookHeaders = splitAtCommas(os.Getenv("NOTIFY_WEBHOOK_HEADERS")) // env value may have comma inside "", parsed separately + } + + whParams := notify.WebhookParams{ + WebhookURL: s.Notify.Webhook.WebhookURL, + Template: s.Notify.Webhook.Template, + Headers: webhookHeaders, + } + webhook, err := notify.NewWebhook(client, whParams) + if err != nil { + return nil, "", errors.Wrap(err, "failed to create webhook notification destination") + } + destinations = append(destinations, webhook) + } + if contains("slack", s.Notify.Admins) { slack, err := notify.NewSlack(s.Notify.Slack.Token, s.Notify.Slack.Channel) if err != nil { @@ -1091,6 +1116,49 @@ func (s *ServerCommand) parseSameSite(ss string) http.SameSite { } } +// splitAtCommas split s at commas, ignoring commas in strings. +// Eliminate leading and trailing dbl quotes in each element only if both presented +// based on https://stackoverflow.com/a/59318708 +func splitAtCommas(s string) []string { + + cleanup := func(s string) string { + if s == "" { + return s + } + res := strings.TrimSpace(s) + if res[0] == '"' && res[len(res)-1] == '"' { + res = strings.TrimPrefix(res, `"`) + res = strings.TrimSuffix(res, `"`) + } + return res + } + + var res []string + var beg int + var inString bool + + for i := 0; i < len(s); i++ { + if s[i] == ',' && !inString { + res = append(res, cleanup(s[beg:i])) + beg = i + 1 + continue + } + + if s[i] == '"' { + if !inString { + inString = true + } else if i > 0 && s[i-1] != '\\' { // also allow \" + inString = false + } + } + } + res = append(res, cleanup(s[beg:])) + if len(res) == 1 && res[0] == "" { + return []string{} + } + return res +} + // authRefreshCache used by authenticator to minimize repeatable token refreshes type authRefreshCache struct { cache.LoadingCache diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index bb31ebb5..ea93d78a 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -619,6 +619,29 @@ func TestServerCommand_parseSameSite(t *testing.T) { } } +func Test_splitAtCommas(t *testing.T) { + + tbl := []struct { + inp string + res []string + }{ + {"a string", []string{"a string"}}, + {"vv1, vv2, vv3", []string{"vv1", "vv2", "vv3"}}, + {`"vv1, blah", vv2, vv3`, []string{"vv1, blah", "vv2", "vv3"}}, + { + `Access-Control-Allow-Headers:"DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type",header123:val, foo:"bar1,bar2"`, + []string{"Access-Control-Allow-Headers:\"DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type\"", "header123:val", "foo:\"bar1,bar2\""}, + }, + {"", []string{}}, + } + + for i, tt := range tbl { + t.Run(strconv.Itoa(i), func(t *testing.T) { + assert.Equal(t, tt.res, splitAtCommas(tt.inp)) + }) + } +} + func chooseRandomUnusedPort() (port int) { for i := 0; i < 10; i++ { port = 40000 + int(rand.Int31n(10000)) diff --git a/backend/app/main_test.go b/backend/app/main_test.go index a7d40f06..232e101b 100644 --- a/backend/app/main_test.go +++ b/backend/app/main_test.go @@ -6,9 +6,11 @@ import ( "math/rand" "net" "net/http" + "net/http/httptest" "os" "strconv" "strings" + "sync/atomic" "syscall" "testing" "time" @@ -57,6 +59,67 @@ func Test_Main(t *testing.T) { assert.Equal(t, "pong", string(body)) } +func TestMain_WithWebhook(t *testing.T) { + dir, err := ioutil.TempDir(os.TempDir(), "remark42") + require.NoError(t, err) + defer os.RemoveAll(dir) + + var webhookSent int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.StoreInt32(&webhookSent, 1) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + + b, e := ioutil.ReadAll(r.Body) + defer r.Body.Close() + + assert.Nil(t, e) + assert.Equal(t, "Comment: env test", string(b)) + })) + defer ts.Close() + + port := chooseRandomUnusedPort() + os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp", + "--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg", + "--admin-passwd=password", "--site=remark", "--notify.admins=webhook"} + + err = os.Setenv("NOTIFY_WEBHOOK_URL", ts.URL) + assert.NoError(t, err) + err = os.Setenv("NOTIFY_WEBHOOK_TEMPLATE", "Comment: {{.Orig}}") + assert.NoError(t, err) + err = os.Setenv("NOTIFY_WEBHOOK_HEADERS", "Content-Type:application/json") + assert.NoError(t, err) + + done := make(chan struct{}) + go func() { + <-done + e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM) + require.NoError(t, e) + }() + + finished := make(chan struct{}) + go func() { + main() + assert.Eventuallyf(t, func() bool { + return atomic.LoadInt32(&webhookSent) == int32(1) + }, time.Second, 100*time.Millisecond, "webhook was not sent") + close(finished) + }() + + // defer cleanup because require check below can fail + defer func() { + close(done) + <-finished + }() + + waitForHTTPServerStart(port) + + resp, err := http.Post(fmt.Sprintf("http://admin:password@localhost:%d/api/v1/comment", port), "", + strings.NewReader(`{"text": "env test", "locator":{"url": "https://radio-t.com", "site": "remark"}}`)) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, 201, resp.StatusCode) +} + func TestGetDump(t *testing.T) { dump := getDump() assert.True(t, strings.Contains(dump, "goroutine")) diff --git a/backend/app/notify/webhook.go b/backend/app/notify/webhook.go new file mode 100644 index 00000000..ce814b59 --- /dev/null +++ b/backend/app/notify/webhook.go @@ -0,0 +1,113 @@ +package notify + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http" + "strings" + "text/template" + + log "github.com/go-pkgz/lgr" + "github.com/pkg/errors" +) + +const ( + webhookDefaultTemplate = `{"text": "{{.Text}}"}` +) + +// WebhookClient defines an interface of client for webhook +type WebhookClient interface { + Do(*http.Request) (*http.Response, error) +} + +// WebhookParams contain settings for webhook notifications +type WebhookParams struct { + WebhookURL string + Template string + Headers []string +} + +// Webhook implements notify.Destination for Webhook notifications +type Webhook struct { + WebhookParams + webhookClient WebhookClient + webhookTemplate *template.Template +} + +// NewWebhook makes Webhook +func NewWebhook(client WebhookClient, params WebhookParams) (*Webhook, error) { + res := &Webhook{WebhookParams: params} + if res.WebhookURL == "" { + return nil, errors.New("webhook URL is required for webhook notifications") + } + + if res.Template == "" { + res.Template = webhookDefaultTemplate + } + + payloadTmpl, err := template.New("webhook").Parse(res.Template) + if err != nil { + return nil, errors.Wrap(err, "unable to parse webhook template") + } + + res.webhookClient = client + res.webhookTemplate = payloadTmpl + + log.Printf("[DEBUG] create new webhook notifier for %s", res.WebhookURL) + + return res, nil +} + +// Send sends Webhook notification +func (t *Webhook) Send(ctx context.Context, req Request) error { + var payload bytes.Buffer + err := t.webhookTemplate.Execute(&payload, req.Comment) + if err != nil { + return errors.Wrap(err, "unable to compile webhook template") + } + + httpReq, err := http.NewRequestWithContext(ctx, "POST", t.WebhookURL, &payload) + if err != nil { + return errors.Wrap(err, "unable to create webhook request") + } + + for _, h := range t.Headers { + elems := strings.Split(h, ":") + if len(elems) != 2 { + continue + } + httpReq.Header.Set(strings.TrimSpace(elems[0]), strings.TrimSpace(elems[1])) + } + + resp, err := t.webhookClient.Do(httpReq) + if err != nil { + return errors.Wrap(err, "webhook request failed") + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("webhook request failed with non-OK status code: %d", resp.StatusCode) + respBody, e := ioutil.ReadAll(resp.Body) + if e != nil { + return fmt.Errorf(errMsg) + } + return fmt.Errorf("%s, body: %s", errMsg, respBody) + } + + log.Printf("[DEBUG] send webhook notification, comment id %s", req.Comment.ID) + + return nil +} + +// SendVerification is not implemented for Webhook +func (t *Webhook) SendVerification(_ context.Context, _ VerificationRequest) error { + return nil +} + +// String describes the webhook instance +func (t *Webhook) String() string { + return fmt.Sprintf("webhook notification to %s", t.WebhookURL) +} diff --git a/backend/app/notify/webhook_test.go b/backend/app/notify/webhook_test.go new file mode 100644 index 00000000..422d7b64 --- /dev/null +++ b/backend/app/notify/webhook_test.go @@ -0,0 +1,160 @@ +package notify + +import ( + "bytes" + "context" + "errors" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark42/backend/app/store" +) + +type funcWebhookClient func(*http.Request) (*http.Response, error) + +func (c funcWebhookClient) Do(r *http.Request) (*http.Response, error) { + return c(r) +} + +var okWebhookClient = funcWebhookClient(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewBufferString("ok")), + }, nil +}) + +type errReader struct { +} + +func (errReader) Read(p []byte) (n int, err error) { + return 0, errors.New("test error") +} + +func TestWebhook_NewWebhook(t *testing.T) { + + wh, err := NewWebhook(okWebhookClient, WebhookParams{ + WebhookURL: "https://example.org/webhook", + Headers: []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="}, + }) + assert.NoError(t, err) + assert.NotNil(t, wh) + + assert.Equal(t, "https://example.org/webhook", wh.WebhookURL) + assert.Equal(t, []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="}, wh.Headers) + assert.Equal(t, `{"text": "{{.Text}}"}`, wh.Template) + + wh, err = NewWebhook(okWebhookClient, WebhookParams{ + WebhookURL: "https://example.org/webhook", + Headers: []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="}, + Template: "{{.Text}}", + }) + assert.NoError(t, err) + assert.NotNil(t, wh) + assert.Equal(t, "{{.Text}}", wh.Template) + + wh, err = NewWebhook(okWebhookClient, WebhookParams{}) + assert.Nil(t, wh) + assert.Error(t, err) + assert.Equal(t, "webhook URL is required for webhook notifications", err.Error()) + + wh, err = NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook", Template: "{{.Text"}) + assert.Nil(t, wh) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unable to parse webhook template") +} + +func TestWebhook_Send(t *testing.T) { + + wh, err := NewWebhook(funcWebhookClient(func(r *http.Request) (*http.Response, error) { + assert.Len(t, r.Header, 1) + assert.Equal(t, r.Header.Get("Content-Type"), "application/json,text/plain") + + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewBufferString("")), + }, nil + }), WebhookParams{ + WebhookURL: "https://example.org/webhook", + Headers: []string{"Content-Type:application/json,text/plain", ""}, + }) + assert.NoError(t, err) + assert.NotNil(t, wh) + + c := store.Comment{Text: "some text", ParentID: "1", ID: "999"} + c.User.Name = "from" + + err = wh.Send(context.TODO(), Request{Comment: c}) + assert.NoError(t, err) + + wh, err = NewWebhook(okWebhookClient, WebhookParams{ + WebhookURL: "https://example.org/webhook", + Template: "{{.InvalidProperty}}", + }) + assert.NoError(t, err) + err = wh.Send(context.TODO(), Request{Comment: c}) + require.Error(t, err) + assert.Contains(t, err.Error(), "webhook template") + + wh, err = NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook"}) + assert.NoError(t, err) + err = wh.Send(nil, Request{Comment: c}) // nolint + require.Error(t, err) + assert.Contains(t, err.Error(), "unable to create webhook request") + + wh, err = NewWebhook(funcWebhookClient(func(*http.Request) (*http.Response, error) { + return nil, errors.New("request failed") + }), WebhookParams{WebhookURL: "https://not-existing-url.net"}) + assert.NoError(t, err) + err = wh.Send(context.TODO(), Request{Comment: c}) + require.Error(t, err) + assert.Contains(t, err.Error(), "webhook request failed") + + wh, err = NewWebhook(funcWebhookClient(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewBufferString("not found")), + }, nil + }), WebhookParams{ + WebhookURL: "http:/example.org/invalid-url", + }) + assert.NoError(t, err) + err = wh.Send(context.TODO(), Request{Comment: c}) + require.Error(t, err) + assert.Contains(t, err.Error(), "non-OK status code: 404, body: not found") + + wh, err = NewWebhook(funcWebhookClient(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(errReader{}), + }, nil + }), WebhookParams{ + WebhookURL: "http:/example.org/invalid-url", + }) + assert.NoError(t, err) + err = wh.Send(context.TODO(), Request{Comment: c}) + require.Error(t, err) + assert.Contains(t, err.Error(), "non-OK status code: 404") + assert.NotContains(t, err.Error(), "body") +} + +func TestWebhook_SendVerification(t *testing.T) { + wh, err := NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook"}) + assert.NoError(t, err) + assert.NotNil(t, wh) + + err = wh.SendVerification(context.TODO(), VerificationRequest{}) + assert.NoError(t, err) +} + +func TestWebhook_String(t *testing.T) { + wh, err := NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook"}) + assert.NoError(t, err) + assert.NotNil(t, wh) + + str := wh.String() + assert.Equal(t, "webhook notification to https://example.org/webhook", str) +}