mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Webhook destination for notifications

Browse Source
This commit is contained in:
bakurin
2021-08-28 12:50:36 -05:00
committed by Umputun
parent 9f6c766919
commit be46e849a4
6 changed files with 435 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7 -3
View File
@@ -14,7 +14,7 @@ Remark42 is a self-hosted, lightweight and simple (yet functional) comment engin
* Images upload with drag-and-drop
* Extractor for recent comments, cross-post
* RSS for all comments and each post
* Telegram, Slack and email notifications for Admins (get notified for each new comment)
* Telegram, Slack, Webhook and email notifications for Admins (get notified for each new comment)
* Email and Telegram notifications for users (get notified when someone responds to your comment)
* Export data to JSON with automatic backups
* No external databases, everything embedded in a single data file
@@ -161,12 +161,16 @@ _this is the recommended way to run Remark42_
| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html` | email content type |
| auth.email.template | AUTH_EMAIL_TEMPLATE | none (predefined) | custom email message template file |
| notify.users | NOTIFY_USERS | none | type of user notifications (Telegram, email) |
| notify.admins | NOTIFY_ADMINS | none | type of admin notifications (Telegram, Slack and/or email) |
| notify.admins | NOTIFY_ADMINS | none | type of admin notifications (Telegram, Slack, webhook and/or email) |
| notify.queue | NOTIFY_QUEUE | `100` | size of notification queue |
| notify.telegram.chan | NOTIFY_TELEGRAM_CHAN | | Telegram channel |
| notify.slack.token | NOTIFY_SLACK_TOKEN | | Slack token |
| notify.slack.chan | NOTIFY_SLACK_CHAN | `general` | Slack channel |
| notify.email.fromAddress | NOTIFY_EMAIL_FROM | | from email address |
| notify.webhook.url | NOTIFY_WEBHOOK_URL | | Webhook notification URL |
| notify.webhook.template | NOTIFY_WEBHOOK_TEMPLATE | `{"text": "{{.Text}}"}` | Webhook payload template |
| notify.webhook.headers | NOTIFY_WEBHOOK_HEADERS | | HTTP header in format Header1:Value1,Header2:Value2,...|
| notify.webhook.timeout | NOTIFY_WEBHOOK_TIMEOUT | `5s` | Webhook connection timeout |
| notify.email.fromAddress| NOTIFY_EMAIL_FROM | | from email address |
| notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject |
| telegram.token | TELEGRAM_TOKEN | | Telegram token (used for auth and Telegram notifications) |
| telegram.timeout | TELEGRAM_TIMEOUT | `5s` | Telegram connection timeout |
backend/app/cmd/server.go
+69 -1
View File
@@ -213,7 +213,7 @@ type SMTPGroup struct {
type NotifyGroup struct {
Type []string `long:"type" env:"TYPE" description:"[deprecated, use user and admin types instead] types of notifications" choice:"none" choice:"telegram" choice:"email" choice:"slack" default:"none" env-delim:","` //nolint
Users []string `long:"users" env:"USERS" description:"types of user notifications" choice:"none" choice:"email" choice:"telegram" default:"none" env-delim:","` //nolint
Admins []string `long:"admins" env:"ADMINS" description:"types of admin notifications" choice:"none" choice:"telegram" choice:"email" choice:"slack" default:"none" env-delim:","` //nolint
Admins []string `long:"admins" env:"ADMINS" description:"types of admin notifications" choice:"none" choice:"telegram" choice:"email" choice:"slack" choice:"webhook" default:"none" env-delim:","` //nolint
QueueSize int `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
Telegram struct {
Channel string `long:"chan" env:"CHAN" description:"telegram channel for admin notifications"`
@@ -230,6 +230,12 @@ type NotifyGroup struct {
Token string `long:"token" env:"TOKEN" description:"slack token"`
Channel string `long:"chan" env:"CHAN" description:"slack channel"`
} `group:"slack" namespace:"slack" env-namespace:"SLACK"`
Webhook struct {
WebhookURL string `long:"url" env:"URL" description:"webhook notification URL"`
Template string `long:"template" env:"TEMPLATE" description:"webhook authentication template" default:"{\"text\": \"{{.Text}}\"}"`
Headers []string `long:"headers" description:"webhook authentication headers in format --notify.webhook.headers=Header1:Value1,Value2,..."` // env NOTIFY_WEBHOOK_HEADERS split in code bellow to allow , inside ""
Timeout time.Duration `long:"timeout" env:"TIMEOUT" description:"webhook timeout" default:"5s"`
} `group:"webhook" namespace:"webhook" env-namespace:"WEBHOOK"`
}
// SSLGroup defines options group for server ssl params
@@ -903,6 +909,25 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
var destinations []notify.Destination
var telegramBotUsername string
if contains("webhook", s.Notify.Admins) {
client := &http.Client{Timeout: 5 * time.Second}
webhookHeaders := s.Notify.Webhook.Headers
if len(webhookHeaders) == 0 {
webhookHeaders = splitAtCommas(os.Getenv("NOTIFY_WEBHOOK_HEADERS")) // env value may have comma inside "", parsed separately
}
whParams := notify.WebhookParams{
WebhookURL: s.Notify.Webhook.WebhookURL,
Template: s.Notify.Webhook.Template,
Headers: webhookHeaders,
}
webhook, err := notify.NewWebhook(client, whParams)
if err != nil {
return nil, "", errors.Wrap(err, "failed to create webhook notification destination")
}
destinations = append(destinations, webhook)
}
if contains("slack", s.Notify.Admins) {
slack, err := notify.NewSlack(s.Notify.Slack.Token, s.Notify.Slack.Channel)
if err != nil {
@@ -1091,6 +1116,49 @@ func (s *ServerCommand) parseSameSite(ss string) http.SameSite {
}
}
// splitAtCommas split s at commas, ignoring commas in strings.
// Eliminate leading and trailing dbl quotes in each element only if both presented
// based on https://stackoverflow.com/a/59318708
func splitAtCommas(s string) []string {
cleanup := func(s string) string {
if s == "" {
return s
}
res := strings.TrimSpace(s)
if res[0] == '"' && res[len(res)-1] == '"' {
res = strings.TrimPrefix(res, `"`)
res = strings.TrimSuffix(res, `"`)
}
return res
}
var res []string
var beg int
var inString bool
for i := 0; i < len(s); i++ {
if s[i] == ',' && !inString {
res = append(res, cleanup(s[beg:i]))
beg = i + 1
continue
}
if s[i] == '"' {
if !inString {
inString = true
} else if i > 0 && s[i-1] != '\\' { // also allow \"
inString = false
}
}
}
res = append(res, cleanup(s[beg:]))
if len(res) == 1 && res[0] == "" {
return []string{}
}
return res
}
// authRefreshCache used by authenticator to minimize repeatable token refreshes
type authRefreshCache struct {
cache.LoadingCache
backend/app/cmd/server_test.go
+23
View File
@@ -619,6 +619,29 @@ func TestServerCommand_parseSameSite(t *testing.T) {
}
}
func Test_splitAtCommas(t *testing.T) {
tbl := []struct {
inp string
res []string
}{
{"a string", []string{"a string"}},
{"vv1, vv2, vv3", []string{"vv1", "vv2", "vv3"}},
{`"vv1, blah", vv2, vv3`, []string{"vv1, blah", "vv2", "vv3"}},
{
`Access-Control-Allow-Headers:"DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type",header123:val, foo:"bar1,bar2"`,
[]string{"Access-Control-Allow-Headers:\"DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type\"", "header123:val", "foo:\"bar1,bar2\""},
},
{"", []string{}},
}
for i, tt := range tbl {
t.Run(strconv.Itoa(i), func(t *testing.T) {
assert.Equal(t, tt.res, splitAtCommas(tt.inp))
})
}
}
func chooseRandomUnusedPort() (port int) {
for i := 0; i < 10; i++ {
port = 40000 + int(rand.Int31n(10000))
backend/app/main_test.go
+63
View File
@@ -6,9 +6,11 @@ import (
"math/rand"
"net"
"net/http"
"net/http/httptest"
"os"
"strconv"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
@@ -57,6 +59,67 @@ func Test_Main(t *testing.T) {
assert.Equal(t, "pong", string(body))
}
func TestMain_WithWebhook(t *testing.T) {
dir, err := ioutil.TempDir(os.TempDir(), "remark42")
require.NoError(t, err)
defer os.RemoveAll(dir)
var webhookSent int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.StoreInt32(&webhookSent, 1)
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
b, e := ioutil.ReadAll(r.Body)
defer r.Body.Close()
assert.Nil(t, e)
assert.Equal(t, "Comment: env test", string(b))
}))
defer ts.Close()
port := chooseRandomUnusedPort()
os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg",
"--admin-passwd=password", "--site=remark", "--notify.admins=webhook"}
err = os.Setenv("NOTIFY_WEBHOOK_URL", ts.URL)
assert.NoError(t, err)
err = os.Setenv("NOTIFY_WEBHOOK_TEMPLATE", "Comment: {{.Orig}}")
assert.NoError(t, err)
err = os.Setenv("NOTIFY_WEBHOOK_HEADERS", "Content-Type:application/json")
assert.NoError(t, err)
done := make(chan struct{})
go func() {
<-done
e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
require.NoError(t, e)
}()
finished := make(chan struct{})
go func() {
main()
assert.Eventuallyf(t, func() bool {
return atomic.LoadInt32(&webhookSent) == int32(1)
}, time.Second, 100*time.Millisecond, "webhook was not sent")
close(finished)
}()
// defer cleanup because require check below can fail
defer func() {
close(done)
<-finished
}()
waitForHTTPServerStart(port)
resp, err := http.Post(fmt.Sprintf("http://admin:password@localhost:%d/api/v1/comment", port), "",
strings.NewReader(`{"text": "env test", "locator":{"url": "https://radio-t.com", "site": "remark"}}`))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, 201, resp.StatusCode)
}
func TestGetDump(t *testing.T) {
dump := getDump()
assert.True(t, strings.Contains(dump, "goroutine"))
backend/app/notify/webhook.go
+113
View File
@@ -0,0 +1,113 @@
package notify
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"net/http"
"strings"
"text/template"
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
)
const (
webhookDefaultTemplate = `{"text": "{{.Text}}"}`
)
// WebhookClient defines an interface of client for webhook
type WebhookClient interface {
Do(*http.Request) (*http.Response, error)
}
// WebhookParams contain settings for webhook notifications
type WebhookParams struct {
WebhookURL string
Template string
Headers []string
}
// Webhook implements notify.Destination for Webhook notifications
type Webhook struct {
WebhookParams
webhookClient WebhookClient
webhookTemplate *template.Template
}
// NewWebhook makes Webhook
func NewWebhook(client WebhookClient, params WebhookParams) (*Webhook, error) {
res := &Webhook{WebhookParams: params}
if res.WebhookURL == "" {
return nil, errors.New("webhook URL is required for webhook notifications")
}
if res.Template == "" {
res.Template = webhookDefaultTemplate
}
payloadTmpl, err := template.New("webhook").Parse(res.Template)
if err != nil {
return nil, errors.Wrap(err, "unable to parse webhook template")
}
res.webhookClient = client
res.webhookTemplate = payloadTmpl
log.Printf("[DEBUG] create new webhook notifier for %s", res.WebhookURL)
return res, nil
}
// Send sends Webhook notification
func (t *Webhook) Send(ctx context.Context, req Request) error {
var payload bytes.Buffer
err := t.webhookTemplate.Execute(&payload, req.Comment)
if err != nil {
return errors.Wrap(err, "unable to compile webhook template")
}
httpReq, err := http.NewRequestWithContext(ctx, "POST", t.WebhookURL, &payload)
if err != nil {
return errors.Wrap(err, "unable to create webhook request")
}
for _, h := range t.Headers {
elems := strings.Split(h, ":")
if len(elems) != 2 {
continue
}
httpReq.Header.Set(strings.TrimSpace(elems[0]), strings.TrimSpace(elems[1]))
}
resp, err := t.webhookClient.Do(httpReq)
if err != nil {
return errors.Wrap(err, "webhook request failed")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
errMsg := fmt.Sprintf("webhook request failed with non-OK status code: %d", resp.StatusCode)
respBody, e := ioutil.ReadAll(resp.Body)
if e != nil {
return fmt.Errorf(errMsg)
}
return fmt.Errorf("%s, body: %s", errMsg, respBody)
}
log.Printf("[DEBUG] send webhook notification, comment id %s", req.Comment.ID)
return nil
}
// SendVerification is not implemented for Webhook
func (t *Webhook) SendVerification(_ context.Context, _ VerificationRequest) error {
return nil
}
// String describes the webhook instance
func (t *Webhook) String() string {
return fmt.Sprintf("webhook notification to %s", t.WebhookURL)
}
backend/app/notify/webhook_test.go
+160
View File
@@ -0,0 +1,160 @@
package notify
import (
"bytes"
"context"
"errors"
"io/ioutil"
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
)
type funcWebhookClient func(*http.Request) (*http.Response, error)
func (c funcWebhookClient) Do(r *http.Request) (*http.Response, error) {
return c(r)
}
var okWebhookClient = funcWebhookClient(func(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(bytes.NewBufferString("ok")),
}, nil
})
type errReader struct {
}
func (errReader) Read(p []byte) (n int, err error) {
return 0, errors.New("test error")
}
func TestWebhook_NewWebhook(t *testing.T) {
wh, err := NewWebhook(okWebhookClient, WebhookParams{
WebhookURL: "https://example.org/webhook",
Headers: []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="},
})
assert.NoError(t, err)
assert.NotNil(t, wh)
assert.Equal(t, "https://example.org/webhook", wh.WebhookURL)
assert.Equal(t, []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="}, wh.Headers)
assert.Equal(t, `{"text": "{{.Text}}"}`, wh.Template)
wh, err = NewWebhook(okWebhookClient, WebhookParams{
WebhookURL: "https://example.org/webhook",
Headers: []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="},
Template: "{{.Text}}",
})
assert.NoError(t, err)
assert.NotNil(t, wh)
assert.Equal(t, "{{.Text}}", wh.Template)
wh, err = NewWebhook(okWebhookClient, WebhookParams{})
assert.Nil(t, wh)
assert.Error(t, err)
assert.Equal(t, "webhook URL is required for webhook notifications", err.Error())
wh, err = NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook", Template: "{{.Text"})
assert.Nil(t, wh)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unable to parse webhook template")
}
func TestWebhook_Send(t *testing.T) {
wh, err := NewWebhook(funcWebhookClient(func(r *http.Request) (*http.Response, error) {
assert.Len(t, r.Header, 1)
assert.Equal(t, r.Header.Get("Content-Type"), "application/json,text/plain")
return &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(bytes.NewBufferString("")),
}, nil
}), WebhookParams{
WebhookURL: "https://example.org/webhook",
Headers: []string{"Content-Type:application/json,text/plain", ""},
})
assert.NoError(t, err)
assert.NotNil(t, wh)
c := store.Comment{Text: "some text", ParentID: "1", ID: "999"}
c.User.Name = "from"
err = wh.Send(context.TODO(), Request{Comment: c})
assert.NoError(t, err)
wh, err = NewWebhook(okWebhookClient, WebhookParams{
WebhookURL: "https://example.org/webhook",
Template: "{{.InvalidProperty}}",
})
assert.NoError(t, err)
err = wh.Send(context.TODO(), Request{Comment: c})
require.Error(t, err)
assert.Contains(t, err.Error(), "webhook template")
wh, err = NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook"})
assert.NoError(t, err)
err = wh.Send(nil, Request{Comment: c}) // nolint
require.Error(t, err)
assert.Contains(t, err.Error(), "unable to create webhook request")
wh, err = NewWebhook(funcWebhookClient(func(*http.Request) (*http.Response, error) {
return nil, errors.New("request failed")
}), WebhookParams{WebhookURL: "https://not-existing-url.net"})
assert.NoError(t, err)
err = wh.Send(context.TODO(), Request{Comment: c})
require.Error(t, err)
assert.Contains(t, err.Error(), "webhook request failed")
wh, err = NewWebhook(funcWebhookClient(func(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusNotFound,
Body: ioutil.NopCloser(bytes.NewBufferString("not found")),
}, nil
}), WebhookParams{
WebhookURL: "http:/example.org/invalid-url",
})
assert.NoError(t, err)
err = wh.Send(context.TODO(), Request{Comment: c})
require.Error(t, err)
assert.Contains(t, err.Error(), "non-OK status code: 404, body: not found")
wh, err = NewWebhook(funcWebhookClient(func(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusNotFound,
Body: ioutil.NopCloser(errReader{}),
}, nil
}), WebhookParams{
WebhookURL: "http:/example.org/invalid-url",
})
assert.NoError(t, err)
err = wh.Send(context.TODO(), Request{Comment: c})
require.Error(t, err)
assert.Contains(t, err.Error(), "non-OK status code: 404")
assert.NotContains(t, err.Error(), "body")
}
func TestWebhook_SendVerification(t *testing.T) {
wh, err := NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook"})
assert.NoError(t, err)
assert.NotNil(t, wh)
err = wh.SendVerification(context.TODO(), VerificationRequest{})
assert.NoError(t, err)
}
func TestWebhook_String(t *testing.T) {
wh, err := NewWebhook(okWebhookClient, WebhookParams{WebhookURL: "https://example.org/webhook"})
assert.NoError(t, err)
assert.NotNil(t, wh)
str := wh.String()
assert.Equal(t, "webhook notification to https://example.org/webhook", str)
}