From b6ab57791eb0eced578e7cc54c307fc8ec69455d Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Thu, 8 Dec 2016 14:40:08 -0800 Subject: [PATCH] Enable sentry reporting. (#180) This commit adds basic sentry reporting. If enabled by setting the appropriate configuration value, it will report panics and errors. Certain functions in the core package (Delegate, Encrypt, Decrypt, Restore, and ResetPersisted) have additional Sentry reporting as these are the most common errors. --- config/config.go | 7 ++++ config/config_test.go | 8 +++++ core/core.go | 25 ++++++++++++++ redoctober.go | 29 ++++++++++++++-- report/report.go | 78 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 report/report.go diff --git a/config/config.go b/config/config.go index e44eb0f..b795ad9 100644 --- a/config/config.go +++ b/config/config.go @@ -73,6 +73,11 @@ type Metrics struct { Port string `json:"port"` } +// Reporting contains configuration for error reporting. +type Reporting struct { + SentryDSN string `json:"sentry_dsn"` +} + // Delegations contains configuration for persisting delegations. type Delegations struct { // Persist controls whether delegations are persisted or not. @@ -100,6 +105,7 @@ type Config struct { UI *UI `json:"ui"` HipChat *HipChat `json:"hipchat"` Metrics *Metrics `json:"metrics"` + Reporting *Reporting `json:"reporting"` Delegations *Delegations `json:"delegations"` } @@ -126,6 +132,7 @@ func New() *Config { UI: &UI{}, HipChat: &HipChat{}, Metrics: &Metrics{}, + Reporting: &Reporting{}, Delegations: &Delegations{}, } } diff --git a/config/config_test.go b/config/config_test.go index 2b5641d..7991cf1 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -62,6 +62,10 @@ func (m *Metrics) equal(other *Metrics) bool { return m.Host == other.Host && m.Port == other.Port } +func (r *Reporting) equal(other *Reporting) bool { + return r.SentryDSN == other.SentryDSN +} + func (d *Delegations) equal(other *Delegations) bool { return d.Persist == other.Persist && d.Policy == other.Policy } @@ -83,6 +87,10 @@ func (c *Config) equal(other *Config) bool { return false } + if !c.Reporting.equal(other.Reporting) { + return false + } + if !c.Delegations.equal(other.Delegations) { return false } diff --git a/core/core.go b/core/core.go index 3040910..be6fafc 100644 --- a/core/core.go +++ b/core/core.go @@ -19,6 +19,7 @@ import ( "github.com/cloudflare/redoctober/order" "github.com/cloudflare/redoctober/passvault" "github.com/cloudflare/redoctober/persist" + "github.com/cloudflare/redoctober/report" ) var ( @@ -233,8 +234,11 @@ func validateName(name, password string) error { func Init(path string, config *config.Config) error { var err error + tags := map[string]string{"function": "core.Init"} + defer func() { if err != nil { + report.Check(err, tags) log.Printf("core.init failed: %v", err) } else { log.Printf("core.init success: path=%s", path) @@ -395,9 +399,15 @@ func Purge(jsonIn []byte) ([]byte, error) { func Delegate(jsonIn []byte) ([]byte, error) { var s DelegateRequest var err error + var tags = map[string]string{"function": "core.Delegate"} defer func() { if err != nil { + tags["delegation.name"] = s.Name + tags["delegation.uses"] = fmt.Sprintf("%d", s.Uses) + tags["delegation.time"] = s.Time + tags["delegation.users"] = strings.Join(s.Users, ", ") + tags["delegation.labels"] = strings.Join(s.Labels, ", ") log.Printf("core.delegate failed: user=%s %v", s.Name, err) } else { log.Printf("core.delegate success: user=%s uses=%d time=%s users=%v labels=%v", s.Name, s.Uses, s.Time, s.Users, s.Labels) @@ -562,9 +572,13 @@ func Password(jsonIn []byte) ([]byte, error) { func Encrypt(jsonIn []byte) ([]byte, error) { var s EncryptRequest var err error + var tags = map[string]string{"function": "core.Encrypt"} defer func() { if err != nil { + tags["encrypt.user"] = s.Name + tags["encrypt.size"] = fmt.Sprintf("%d", len(s.Data)) + report.Check(err, tags) log.Printf("core.encrypt failed: user=%s size=%d %v", s.Name, len(s.Data), err) } else { log.Printf("core.encrypt success: user=%s size=%d", s.Name, len(s.Data)) @@ -644,9 +658,12 @@ func ReEncrypt(jsonIn []byte) ([]byte, error) { func Decrypt(jsonIn []byte) ([]byte, error) { var s DecryptRequest var err error + var tags = map[string]string{"function": "core.Decrypt"} defer func() { if err != nil { + tags["decrypt.user"] = s.Name + report.Check(err, tags) log.Printf("core.decrypt failed: user=%s %v", s.Name, err) } else { log.Printf("core.decrypt success: user=%s", s.Name) @@ -674,6 +691,8 @@ func Decrypt(jsonIn []byte) ([]byte, error) { Delegates: names, } + tags["delegates"] = strings.Join(names, ", ") + out, err := json.Marshal(resp) if err != nil { return jsonStatusError(err) @@ -991,9 +1010,12 @@ func Status(jsonIn []byte) (out []byte, err error) { // Restore attempts a restoration of the persistence store. func Restore(jsonIn []byte) (out []byte, err error) { var req DelegateRequest + var tags = map[string]string{"function": "core.Restore"} defer func() { if err != nil { + tags["restore.user"] = req.Name + report.Check(err, tags) log.Printf("core.restore failed: user=%s %v", req.Name, err) } else { log.Printf("core.restore success: user=%s", req.Name) @@ -1026,9 +1048,12 @@ func Restore(jsonIn []byte) (out []byte, err error) { // request requires an admin. func ResetPersisted(jsonIn []byte) (out []byte, err error) { var req PurgeRequest + var tags = map[string]string{"function": "core.ResetPersisted"} defer func() { if err != nil { + tags["reset-persisted.user"] = req.Name + report.Check(err, tags) log.Printf("core.resetpersisted failed: user=%s %v", req.Name, err) } else { log.Printf("core.resetpersisted success: user=%s", req.Name) diff --git a/redoctober.go b/redoctober.go index 1d0f7e3..185accc 100644 --- a/redoctober.go +++ b/redoctober.go @@ -9,6 +9,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/pem" + "errors" "flag" "fmt" "io" @@ -22,6 +23,7 @@ import ( "github.com/cloudflare/redoctober/config" "github.com/cloudflare/redoctober/core" + "github.com/cloudflare/redoctober/report" "github.com/coreos/go-systemd/activation" "github.com/prometheus/client_golang/prometheus" ) @@ -67,20 +69,29 @@ func processRequest(requestType string, w http.ResponseWriter, r *http.Request) header.Set("Content-Type", "application/json") header.Set("Strict-Transport-Security", "max-age=86400; includeSubDomains; preload") + tags := map[string]string{ + "request-type": requestType, + "request-from": r.RemoteAddr, + } fn, ok := functions[requestType] if !ok { + err := errors.New("redoctober: unknown request for " + requestType) + report.Check(err, tags) http.Error(w, "Unknown request", http.StatusInternalServerError) return } body, err := ioutil.ReadAll(r.Body) if err != nil { + report.Check(err, tags) http.Error(w, err.Error(), http.StatusInternalServerError) return } resp, err := fn(body) if err != nil { + // The function should also report errors in more detail. + report.Check(err, tags) log.Printf("http.main failed: %s: %s", requestType, err) http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -188,9 +199,13 @@ type indexHandler struct { func (this *indexHandler) handle(w http.ResponseWriter, r *http.Request) { var body io.ReadSeeker + var tags = map[string]string{} + if this.staticPath != "" { + tags["static-path"] = this.staticPath f, err := os.Open(this.staticPath) if err != nil { + report.Check(err, tags) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -219,7 +234,9 @@ func initPrometheus() { log.Printf("metrics.init start: addr=%s", srv.Addr) go func() { - log.Fatal(srv.ListenAndServe()) + err := srv.ListenAndServe() + report.Check(err, nil) + log.Fatal(err.Error()) }() } @@ -304,6 +321,8 @@ func main() { cfg = cli } + report.Init(cfg) + if vaultPath == "" || !cfg.Valid() { if !cfg.Valid() { fmt.Fprintf(os.Stderr, "Invalid config.\n") @@ -314,6 +333,7 @@ func main() { } if err := core.Init(vaultPath, cfg); err != nil { + report.Check(err, nil) log.Fatal(err) } @@ -323,9 +343,14 @@ func main() { s, l, err := NewServer(cfg.UI.Static, cfg.Server.Addr, cfg.Server.CAPath, cpaths, kpaths, cfg.Server.Systemd) if err != nil { + report.Check(err, nil) log.Fatalf("Error starting redoctober server: %s\n", err) } log.Printf("http.serve start: addr=%s", cfg.Server.Addr) - log.Fatal(s.Serve(l)) + report.Recover(func() { + err := s.Serve(l) + report.Check(err, nil) + log.Fatal(err.Error()) + }) } diff --git a/report/report.go b/report/report.go new file mode 100644 index 0000000..369ed33 --- /dev/null +++ b/report/report.go @@ -0,0 +1,78 @@ +// Package report contains error reporting functions. +package report + +import ( + "fmt" + "time" + + "github.com/cloudflare/redoctober/config" + raven "github.com/getsentry/raven-go" +) + +// sentry will be set to true if sentry reporting is valid. +var sentry bool + +// sentryTags contains additional tags that can be sent to Sentry. +var sentryTags = map[string]string{} + +func configSentry(cfg *config.Config) { + raven.SetDSN(cfg.Reporting.SentryDSN) + sentry = true + sentryTags["started_at"] = fmt.Sprintf("%d", time.Now().Unix()) + + sentryTags["server.systemd"] = fmt.Sprintf("%v", cfg.Server.Systemd) + if cfg.Server.Addr != "" { + sentryTags["server.address"] = cfg.Server.Addr + } + + sentryTags["metrics.host"] = cfg.Metrics.Host + sentryTags["metrics.port"] = cfg.Metrics.Port + + if cfg.HipChat.ID != "" { + sentryTags["hipchat.id"] = cfg.HipChat.ID + } + + sentryTags["persist.enabled"] = fmt.Sprintf("%v", cfg.Delegations.Persist) + if cfg.Delegations.Persist { + sentryTags["persist.mechanism"] = cfg.Delegations.Mechanism + sentryTags["persist.location"] = cfg.Delegations.Location + } +} + +func Init(cfg *config.Config) { + if cfg.Reporting.SentryDSN != "" { + configSentry(cfg) + } +} + +// Check will see if err contains an error; if it does, and if +// reporting is configured, it will report the error. +func Check(err error, tags map[string]string) { + if err == nil { + return + } + + if tags == nil { + tags = map[string]string{} + } + + if sentry { + for k, v := range sentryTags { + tags[k] = v + } + raven.CaptureError(err, tags) + } +} + +// Recover will wrap the function in a manner that will capture +// panics. If error reporting isn't active, it will just panic. This +// default behaviour allows the stack trace to be capture in the +// system logs and the service management system (e.g. systemd) to +// automatically restart the server. +func Recover(fn func()) { + if sentry { + raven.CapturePanic(fn, sentryTags) + } else { + fn() + } +}