diff --git a/config/config.go b/config/config.go index e44eb0f..b795ad9 100644 --- a/config/config.go +++ b/config/config.go @@ -73,6 +73,11 @@ type Metrics struct { Port string `json:"port"` } +// Reporting contains configuration for error reporting. +type Reporting struct { + SentryDSN string `json:"sentry_dsn"` +} + // Delegations contains configuration for persisting delegations. type Delegations struct { // Persist controls whether delegations are persisted or not. @@ -100,6 +105,7 @@ type Config struct { UI *UI `json:"ui"` HipChat *HipChat `json:"hipchat"` Metrics *Metrics `json:"metrics"` + Reporting *Reporting `json:"reporting"` Delegations *Delegations `json:"delegations"` } @@ -126,6 +132,7 @@ func New() *Config { UI: &UI{}, HipChat: &HipChat{}, Metrics: &Metrics{}, + Reporting: &Reporting{}, Delegations: &Delegations{}, } } diff --git a/config/config_test.go b/config/config_test.go index 2b5641d..7991cf1 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -62,6 +62,10 @@ func (m *Metrics) equal(other *Metrics) bool { return m.Host == other.Host && m.Port == other.Port } +func (r *Reporting) equal(other *Reporting) bool { + return r.SentryDSN == other.SentryDSN +} + func (d *Delegations) equal(other *Delegations) bool { return d.Persist == other.Persist && d.Policy == other.Policy } @@ -83,6 +87,10 @@ func (c *Config) equal(other *Config) bool { return false } + if !c.Reporting.equal(other.Reporting) { + return false + } + if !c.Delegations.equal(other.Delegations) { return false } diff --git a/core/core.go b/core/core.go index 3040910..be6fafc 100644 --- a/core/core.go +++ b/core/core.go @@ -19,6 +19,7 @@ import ( "github.com/cloudflare/redoctober/order" "github.com/cloudflare/redoctober/passvault" "github.com/cloudflare/redoctober/persist" + "github.com/cloudflare/redoctober/report" ) var ( @@ -233,8 +234,11 @@ func validateName(name, password string) error { func Init(path string, config *config.Config) error { var err error + tags := map[string]string{"function": "core.Init"} + defer func() { if err != nil { + report.Check(err, tags) log.Printf("core.init failed: %v", err) } else { log.Printf("core.init success: path=%s", path) @@ -395,9 +399,15 @@ func Purge(jsonIn []byte) ([]byte, error) { func Delegate(jsonIn []byte) ([]byte, error) { var s DelegateRequest var err error + var tags = map[string]string{"function": "core.Delegate"} defer func() { if err != nil { + tags["delegation.name"] = s.Name + tags["delegation.uses"] = fmt.Sprintf("%d", s.Uses) + tags["delegation.time"] = s.Time + tags["delegation.users"] = strings.Join(s.Users, ", ") + tags["delegation.labels"] = strings.Join(s.Labels, ", ") log.Printf("core.delegate failed: user=%s %v", s.Name, err) } else { log.Printf("core.delegate success: user=%s uses=%d time=%s users=%v labels=%v", s.Name, s.Uses, s.Time, s.Users, s.Labels) @@ -562,9 +572,13 @@ func Password(jsonIn []byte) ([]byte, error) { func Encrypt(jsonIn []byte) ([]byte, error) { var s EncryptRequest var err error + var tags = map[string]string{"function": "core.Encrypt"} defer func() { if err != nil { + tags["encrypt.user"] = s.Name + tags["encrypt.size"] = fmt.Sprintf("%d", len(s.Data)) + report.Check(err, tags) log.Printf("core.encrypt failed: user=%s size=%d %v", s.Name, len(s.Data), err) } else { log.Printf("core.encrypt success: user=%s size=%d", s.Name, len(s.Data)) @@ -644,9 +658,12 @@ func ReEncrypt(jsonIn []byte) ([]byte, error) { func Decrypt(jsonIn []byte) ([]byte, error) { var s DecryptRequest var err error + var tags = map[string]string{"function": "core.Decrypt"} defer func() { if err != nil { + tags["decrypt.user"] = s.Name + report.Check(err, tags) log.Printf("core.decrypt failed: user=%s %v", s.Name, err) } else { log.Printf("core.decrypt success: user=%s", s.Name) @@ -674,6 +691,8 @@ func Decrypt(jsonIn []byte) ([]byte, error) { Delegates: names, } + tags["delegates"] = strings.Join(names, ", ") + out, err := json.Marshal(resp) if err != nil { return jsonStatusError(err) @@ -991,9 +1010,12 @@ func Status(jsonIn []byte) (out []byte, err error) { // Restore attempts a restoration of the persistence store. func Restore(jsonIn []byte) (out []byte, err error) { var req DelegateRequest + var tags = map[string]string{"function": "core.Restore"} defer func() { if err != nil { + tags["restore.user"] = req.Name + report.Check(err, tags) log.Printf("core.restore failed: user=%s %v", req.Name, err) } else { log.Printf("core.restore success: user=%s", req.Name) @@ -1026,9 +1048,12 @@ func Restore(jsonIn []byte) (out []byte, err error) { // request requires an admin. func ResetPersisted(jsonIn []byte) (out []byte, err error) { var req PurgeRequest + var tags = map[string]string{"function": "core.ResetPersisted"} defer func() { if err != nil { + tags["reset-persisted.user"] = req.Name + report.Check(err, tags) log.Printf("core.resetpersisted failed: user=%s %v", req.Name, err) } else { log.Printf("core.resetpersisted success: user=%s", req.Name) diff --git a/redoctober.go b/redoctober.go index 1d0f7e3..185accc 100644 --- a/redoctober.go +++ b/redoctober.go @@ -9,6 +9,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/pem" + "errors" "flag" "fmt" "io" @@ -22,6 +23,7 @@ import ( "github.com/cloudflare/redoctober/config" "github.com/cloudflare/redoctober/core" + "github.com/cloudflare/redoctober/report" "github.com/coreos/go-systemd/activation" "github.com/prometheus/client_golang/prometheus" ) @@ -67,20 +69,29 @@ func processRequest(requestType string, w http.ResponseWriter, r *http.Request) header.Set("Content-Type", "application/json") header.Set("Strict-Transport-Security", "max-age=86400; includeSubDomains; preload") + tags := map[string]string{ + "request-type": requestType, + "request-from": r.RemoteAddr, + } fn, ok := functions[requestType] if !ok { + err := errors.New("redoctober: unknown request for " + requestType) + report.Check(err, tags) http.Error(w, "Unknown request", http.StatusInternalServerError) return } body, err := ioutil.ReadAll(r.Body) if err != nil { + report.Check(err, tags) http.Error(w, err.Error(), http.StatusInternalServerError) return } resp, err := fn(body) if err != nil { + // The function should also report errors in more detail. + report.Check(err, tags) log.Printf("http.main failed: %s: %s", requestType, err) http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -188,9 +199,13 @@ type indexHandler struct { func (this *indexHandler) handle(w http.ResponseWriter, r *http.Request) { var body io.ReadSeeker + var tags = map[string]string{} + if this.staticPath != "" { + tags["static-path"] = this.staticPath f, err := os.Open(this.staticPath) if err != nil { + report.Check(err, tags) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -219,7 +234,9 @@ func initPrometheus() { log.Printf("metrics.init start: addr=%s", srv.Addr) go func() { - log.Fatal(srv.ListenAndServe()) + err := srv.ListenAndServe() + report.Check(err, nil) + log.Fatal(err.Error()) }() } @@ -304,6 +321,8 @@ func main() { cfg = cli } + report.Init(cfg) + if vaultPath == "" || !cfg.Valid() { if !cfg.Valid() { fmt.Fprintf(os.Stderr, "Invalid config.\n") @@ -314,6 +333,7 @@ func main() { } if err := core.Init(vaultPath, cfg); err != nil { + report.Check(err, nil) log.Fatal(err) } @@ -323,9 +343,14 @@ func main() { s, l, err := NewServer(cfg.UI.Static, cfg.Server.Addr, cfg.Server.CAPath, cpaths, kpaths, cfg.Server.Systemd) if err != nil { + report.Check(err, nil) log.Fatalf("Error starting redoctober server: %s\n", err) } log.Printf("http.serve start: addr=%s", cfg.Server.Addr) - log.Fatal(s.Serve(l)) + report.Recover(func() { + err := s.Serve(l) + report.Check(err, nil) + log.Fatal(err.Error()) + }) } diff --git a/report/report.go b/report/report.go new file mode 100644 index 0000000..369ed33 --- /dev/null +++ b/report/report.go @@ -0,0 +1,78 @@ +// Package report contains error reporting functions. +package report + +import ( + "fmt" + "time" + + "github.com/cloudflare/redoctober/config" + raven "github.com/getsentry/raven-go" +) + +// sentry will be set to true if sentry reporting is valid. +var sentry bool + +// sentryTags contains additional tags that can be sent to Sentry. +var sentryTags = map[string]string{} + +func configSentry(cfg *config.Config) { + raven.SetDSN(cfg.Reporting.SentryDSN) + sentry = true + sentryTags["started_at"] = fmt.Sprintf("%d", time.Now().Unix()) + + sentryTags["server.systemd"] = fmt.Sprintf("%v", cfg.Server.Systemd) + if cfg.Server.Addr != "" { + sentryTags["server.address"] = cfg.Server.Addr + } + + sentryTags["metrics.host"] = cfg.Metrics.Host + sentryTags["metrics.port"] = cfg.Metrics.Port + + if cfg.HipChat.ID != "" { + sentryTags["hipchat.id"] = cfg.HipChat.ID + } + + sentryTags["persist.enabled"] = fmt.Sprintf("%v", cfg.Delegations.Persist) + if cfg.Delegations.Persist { + sentryTags["persist.mechanism"] = cfg.Delegations.Mechanism + sentryTags["persist.location"] = cfg.Delegations.Location + } +} + +func Init(cfg *config.Config) { + if cfg.Reporting.SentryDSN != "" { + configSentry(cfg) + } +} + +// Check will see if err contains an error; if it does, and if +// reporting is configured, it will report the error. +func Check(err error, tags map[string]string) { + if err == nil { + return + } + + if tags == nil { + tags = map[string]string{} + } + + if sentry { + for k, v := range sentryTags { + tags[k] = v + } + raven.CaptureError(err, tags) + } +} + +// Recover will wrap the function in a manner that will capture +// panics. If error reporting isn't active, it will just panic. This +// default behaviour allows the stack trace to be capture in the +// system logs and the service management system (e.g. systemd) to +// automatically restart the server. +func Recover(fn func()) { + if sentry { + raven.CapturePanic(fn, sentryTags) + } else { + fn() + } +}