Enable sentry reporting. (#180)

This commit adds basic sentry reporting. If enabled by setting the
appropriate configuration value, it will report panics and errors.
Certain functions in the core package (Delegate, Encrypt, Decrypt,
Restore, and ResetPersisted) have additional Sentry reporting as
these are the most common errors.

config/config.go

@@ -73,6 +73,11 @@ type Metrics struct {
 	Port string `json:"port"`
 }
 
+// Reporting contains configuration for error reporting.
+type Reporting struct {
+	SentryDSN string `json:"sentry_dsn"`
+}
+
 // Delegations contains configuration for persisting delegations.
 type Delegations struct {
 	// Persist controls whether delegations are persisted or not.
@@ -100,6 +105,7 @@ type Config struct {
 	UI          *UI          `json:"ui"`
 	HipChat     *HipChat     `json:"hipchat"`
 	Metrics     *Metrics     `json:"metrics"`
+	Reporting   *Reporting   `json:"reporting"`
 	Delegations *Delegations `json:"delegations"`
 }
 
@@ -126,6 +132,7 @@ func New() *Config {
 		UI:          &UI{},
 		HipChat:     &HipChat{},
 		Metrics:     &Metrics{},
+		Reporting:   &Reporting{},
 		Delegations: &Delegations{},
 	}
 }

config/config_test.go

@@ -62,6 +62,10 @@ func (m *Metrics) equal(other *Metrics) bool {
 	return m.Host == other.Host && m.Port == other.Port
 }
 
+func (r *Reporting) equal(other *Reporting) bool {
+	return r.SentryDSN == other.SentryDSN
+}
+
 func (d *Delegations) equal(other *Delegations) bool {
 	return d.Persist == other.Persist && d.Policy == other.Policy
 }
@@ -83,6 +87,10 @@ func (c *Config) equal(other *Config) bool {
 		return false
 	}
 
+	if !c.Reporting.equal(other.Reporting) {
+		return false
+	}
+
 	if !c.Delegations.equal(other.Delegations) {
 		return false
 	}

core/core.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cloudflare/redoctober/order"
 	"github.com/cloudflare/redoctober/passvault"
 	"github.com/cloudflare/redoctober/persist"
+	"github.com/cloudflare/redoctober/report"
 )
 
 var (
@@ -233,8 +234,11 @@ func validateName(name, password string) error {
 func Init(path string, config *config.Config) error {
 	var err error
 
+	tags := map[string]string{"function": "core.Init"}
+
 	defer func() {
 		if err != nil {
+			report.Check(err, tags)
 			log.Printf("core.init failed: %v", err)
 		} else {
 			log.Printf("core.init success: path=%s", path)
@@ -395,9 +399,15 @@ func Purge(jsonIn []byte) ([]byte, error) {
 func Delegate(jsonIn []byte) ([]byte, error) {
 	var s DelegateRequest
 	var err error
+	var tags = map[string]string{"function": "core.Delegate"}
 
 	defer func() {
 		if err != nil {
+			tags["delegation.name"] = s.Name
+			tags["delegation.uses"] = fmt.Sprintf("%d", s.Uses)
+			tags["delegation.time"] = s.Time
+			tags["delegation.users"] = strings.Join(s.Users, ", ")
+			tags["delegation.labels"] = strings.Join(s.Labels, ", ")
 			log.Printf("core.delegate failed: user=%s %v", s.Name, err)
 		} else {
 			log.Printf("core.delegate success: user=%s uses=%d time=%s users=%v labels=%v", s.Name, s.Uses, s.Time, s.Users, s.Labels)
@@ -562,9 +572,13 @@ func Password(jsonIn []byte) ([]byte, error) {
 func Encrypt(jsonIn []byte) ([]byte, error) {
 	var s EncryptRequest
 	var err error
+	var tags = map[string]string{"function": "core.Encrypt"}
 
 	defer func() {
 		if err != nil {
+			tags["encrypt.user"] = s.Name
+			tags["encrypt.size"] = fmt.Sprintf("%d", len(s.Data))
+			report.Check(err, tags)
 			log.Printf("core.encrypt failed: user=%s size=%d %v", s.Name, len(s.Data), err)
 		} else {
 			log.Printf("core.encrypt success: user=%s size=%d", s.Name, len(s.Data))
@@ -644,9 +658,12 @@ func ReEncrypt(jsonIn []byte) ([]byte, error) {
 func Decrypt(jsonIn []byte) ([]byte, error) {
 	var s DecryptRequest
 	var err error
+	var tags = map[string]string{"function": "core.Decrypt"}
 
 	defer func() {
 		if err != nil {
+			tags["decrypt.user"] = s.Name
+			report.Check(err, tags)
 			log.Printf("core.decrypt failed: user=%s %v", s.Name, err)
 		} else {
 			log.Printf("core.decrypt success: user=%s", s.Name)
@@ -674,6 +691,8 @@ func Decrypt(jsonIn []byte) ([]byte, error) {
 		Delegates: names,
 	}
 
+	tags["delegates"] = strings.Join(names, ", ")
+
 	out, err := json.Marshal(resp)
 	if err != nil {
 		return jsonStatusError(err)
@@ -991,9 +1010,12 @@ func Status(jsonIn []byte) (out []byte, err error) {
 // Restore attempts a restoration of the persistence store.
 func Restore(jsonIn []byte) (out []byte, err error) {
 	var req DelegateRequest
+	var tags = map[string]string{"function": "core.Restore"}
 
 	defer func() {
 		if err != nil {
+			tags["restore.user"] = req.Name
+			report.Check(err, tags)
 			log.Printf("core.restore failed: user=%s %v", req.Name, err)
 		} else {
 			log.Printf("core.restore success: user=%s", req.Name)
@@ -1026,9 +1048,12 @@ func Restore(jsonIn []byte) (out []byte, err error) {
 // request requires an admin.
 func ResetPersisted(jsonIn []byte) (out []byte, err error) {
 	var req PurgeRequest
+	var tags = map[string]string{"function": "core.ResetPersisted"}
 
 	defer func() {
 		if err != nil {
+			tags["reset-persisted.user"] = req.Name
+			report.Check(err, tags)
 			log.Printf("core.resetpersisted failed: user=%s %v", req.Name, err)
 		} else {
 			log.Printf("core.resetpersisted success: user=%s", req.Name)

redoctober.go

@@ -9,6 +9,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -22,6 +23,7 @@ import (
 
 	"github.com/cloudflare/redoctober/config"
 	"github.com/cloudflare/redoctober/core"
+	"github.com/cloudflare/redoctober/report"
 	"github.com/coreos/go-systemd/activation"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -67,20 +69,29 @@ func processRequest(requestType string, w http.ResponseWriter, r *http.Request)
 	header.Set("Content-Type", "application/json")
 	header.Set("Strict-Transport-Security", "max-age=86400; includeSubDomains; preload")
 
+	tags := map[string]string{
+		"request-type": requestType,
+		"request-from": r.RemoteAddr,
+	}
 	fn, ok := functions[requestType]
 	if !ok {
+		err := errors.New("redoctober: unknown request for " + requestType)
+		report.Check(err, tags)
 		http.Error(w, "Unknown request", http.StatusInternalServerError)
 		return
 	}
 
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
+		report.Check(err, tags)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
 	resp, err := fn(body)
 	if err != nil {
+		// The function should also report errors in more detail.
+		report.Check(err, tags)
 		log.Printf("http.main failed: %s: %s", requestType, err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -188,9 +199,13 @@ type indexHandler struct {
 
 func (this *indexHandler) handle(w http.ResponseWriter, r *http.Request) {
 	var body io.ReadSeeker
+	var tags = map[string]string{}
+
 	if this.staticPath != "" {
+		tags["static-path"] = this.staticPath
 		f, err := os.Open(this.staticPath)
 		if err != nil {
+			report.Check(err, tags)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
@@ -219,7 +234,9 @@ func initPrometheus() {
 
 	log.Printf("metrics.init start: addr=%s", srv.Addr)
 	go func() {
-		log.Fatal(srv.ListenAndServe())
+		err := srv.ListenAndServe()
+		report.Check(err, nil)
+		log.Fatal(err.Error())
 	}()
 }
 
@@ -304,6 +321,8 @@ func main() {
 		cfg = cli
 	}
 
+	report.Init(cfg)
+
 	if vaultPath == "" || !cfg.Valid() {
 		if !cfg.Valid() {
 			fmt.Fprintf(os.Stderr, "Invalid config.\n")
@@ -314,6 +333,7 @@ func main() {
 	}
 
 	if err := core.Init(vaultPath, cfg); err != nil {
+		report.Check(err, nil)
 		log.Fatal(err)
 	}
 
@@ -323,9 +343,14 @@ func main() {
 	s, l, err := NewServer(cfg.UI.Static, cfg.Server.Addr, cfg.Server.CAPath,
 		cpaths, kpaths, cfg.Server.Systemd)
 	if err != nil {
+		report.Check(err, nil)
 		log.Fatalf("Error starting redoctober server: %s\n", err)
 	}
 
 	log.Printf("http.serve start: addr=%s", cfg.Server.Addr)
-	log.Fatal(s.Serve(l))
+	report.Recover(func() {
+		err := s.Serve(l)
+		report.Check(err, nil)
+		log.Fatal(err.Error())
+	})
 }

report/report.go

@@ -0,0 +1,78 @@
+// Package report contains error reporting functions.
+package report
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/cloudflare/redoctober/config"
+	raven "github.com/getsentry/raven-go"
+)
+
+// sentry will be set to true if sentry reporting is valid.
+var sentry bool
+
+// sentryTags contains additional tags that can be sent to Sentry.
+var sentryTags = map[string]string{}
+
+func configSentry(cfg *config.Config) {
+	raven.SetDSN(cfg.Reporting.SentryDSN)
+	sentry = true
+	sentryTags["started_at"] = fmt.Sprintf("%d", time.Now().Unix())
+
+	sentryTags["server.systemd"] = fmt.Sprintf("%v", cfg.Server.Systemd)
+	if cfg.Server.Addr != "" {
+		sentryTags["server.address"] = cfg.Server.Addr
+	}
+
+	sentryTags["metrics.host"] = cfg.Metrics.Host
+	sentryTags["metrics.port"] = cfg.Metrics.Port
+
+	if cfg.HipChat.ID != "" {
+		sentryTags["hipchat.id"] = cfg.HipChat.ID
+	}
+
+	sentryTags["persist.enabled"] = fmt.Sprintf("%v", cfg.Delegations.Persist)
+	if cfg.Delegations.Persist {
+		sentryTags["persist.mechanism"] = cfg.Delegations.Mechanism
+		sentryTags["persist.location"] = cfg.Delegations.Location
+	}
+}
+
+func Init(cfg *config.Config) {
+	if cfg.Reporting.SentryDSN != "" {
+		configSentry(cfg)
+	}
+}
+
+// Check will see if err contains an error; if it does, and if
+// reporting is configured, it will report the error.
+func Check(err error, tags map[string]string) {
+	if err == nil {
+		return
+	}
+
+	if tags == nil {
+		tags = map[string]string{}
+	}
+
+	if sentry {
+		for k, v := range sentryTags {
+			tags[k] = v
+		}
+		raven.CaptureError(err, tags)
+	}
+}
+
+// Recover will wrap the function in a manner that will capture
+// panics. If error reporting isn't active, it will just panic. This
+// default behaviour allows the stack trace to be capture in the
+// system logs and the service management system (e.g. systemd) to
+// automatically restart the server.
+func Recover(fn func()) {
+	if sentry {
+		raven.CapturePanic(fn, sentryTags)
+	} else {
+		fn()
+	}
+}