Compare commits

...
Author SHA1 Message Date
Jaypalsinh BaradandGitHub 734c966307 Implement container health alerts (#1679)
* Implement container health alerts, including a database migration for alert names and UI updates.

* Add bulk container alerts and improve alerting logic

* Implement container-specific alerts with new database collections, API endpoints, and UI components.
2026-09-02 13:19:40 -04:00
16 changed files with 1408 additions and 8 deletions
+2
View File
@@ -105,6 +105,8 @@ func NewAlertManager(app hubLike) *AlertManager {
func (am *AlertManager) bindEvents() {
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("container_alerts").BindFunc(updateHistoryOnContainerAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("container_alerts").BindFunc(resolveHistoryOnContainerAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
}
+116
View File
@@ -117,3 +117,119 @@ func DeleteUserAlerts(e *core.RequestEvent) error {
return e.JSON(http.StatusOK, map[string]any{"success": true, "count": numDeleted})
}
// UpsertUserContainerAlerts handles API request to create or update container alerts for a user
// across multiple containers (POST /api/beszel/user-container-alerts)
func UpsertUserContainerAlerts(e *core.RequestEvent) error {
userID := e.Auth.Id
reqData := struct {
Min uint8 `json:"min"`
Value float64 `json:"value"`
Name string `json:"name"`
Systems []string `json:"systems"`
Containers []string `json:"containers"`
Overwrite bool `json:"overwrite"`
}{}
err := e.BindBody(&reqData)
if err != nil || userID == "" || reqData.Name == "" || len(reqData.Systems) == 0 || len(reqData.Containers) == 0 {
return e.BadRequestError("Bad data", err)
}
containerAlertsCollection, err := e.App.FindCachedCollectionByNameOrId("container_alerts")
if err != nil {
return err
}
err = e.App.RunInTransaction(func(txApp core.App) error {
for _, systemId := range reqData.Systems {
for _, containerId := range reqData.Containers {
// find existing matching alert
alertRecord, err := txApp.FindFirstRecordByFilter(containerAlertsCollection,
"system={:system} && container={:container} && name={:name} && user={:user}",
dbx.Params{"system": systemId, "container": containerId, "name": reqData.Name, "user": userID})
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
// skip if alert already exists and overwrite is not set
if !reqData.Overwrite && alertRecord != nil {
continue
}
// create new alert if it doesn't exist
if alertRecord == nil {
alertRecord = core.NewRecord(containerAlertsCollection)
alertRecord.Set("user", userID)
alertRecord.Set("system", systemId)
alertRecord.Set("container", containerId)
alertRecord.Set("name", reqData.Name)
}
alertRecord.Set("value", reqData.Value)
alertRecord.Set("min", reqData.Min)
if err := txApp.SaveNoValidate(alertRecord); err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
return e.JSON(http.StatusOK, map[string]any{"success": true})
}
// DeleteUserContainerAlerts handles API request to delete container alerts for a user
// across multiple containers (DELETE /api/beszel/user-container-alerts)
func DeleteUserContainerAlerts(e *core.RequestEvent) error {
userID := e.Auth.Id
reqData := struct {
AlertName string `json:"name"`
Systems []string `json:"systems"`
Containers []string `json:"containers"`
}{}
err := e.BindBody(&reqData)
if err != nil || userID == "" || reqData.AlertName == "" || len(reqData.Systems) == 0 || len(reqData.Containers) == 0 {
return e.BadRequestError("Bad data", err)
}
var numDeleted uint16
err = e.App.RunInTransaction(func(txApp core.App) error {
for _, systemId := range reqData.Systems {
for _, containerId := range reqData.Containers {
// Find existing alert to delete
alertRecord, err := txApp.FindFirstRecordByFilter("container_alerts",
"system={:system} && container={:container} && name={:name} && user={:user}",
dbx.Params{"system": systemId, "container": containerId, "name": reqData.AlertName, "user": userID})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// alert doesn't exist, continue to next container
continue
}
return err
}
if err := txApp.Delete(alertRecord); err != nil {
return err
}
numDeleted++
}
}
return nil
})
if err != nil {
return err
}
return e.JSON(http.StatusOK, map[string]any{"success": true, "count": numDeleted})
}
+394
View File
@@ -0,0 +1,394 @@
package alerts
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/types"
"github.com/spf13/cast"
)
// ContainerAlertData holds the data for a container alert
type ContainerAlertData struct {
systemRecord *core.Record
containerRecord *core.Record
alertRecord *core.Record
name string
unit string
val float64
threshold float64
triggered bool
time time.Time
count uint8
min uint8
descriptor string // override descriptor in notification body
}
// ContainerAlertStats represents the stats structure for container alerts
type ContainerAlertStats struct {
Cpu float64 `json:"c"`
Mem float64 `json:"m"`
NetworkSent float64 `json:"ns"`
NetworkRecv float64 `json:"nr"`
Status string `json:"s"`
Health string `json:"h"`
}
// HandleContainerAlerts processes container alerts for a system
func (am *AlertManager) HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData) error {
containers := data.Containers
if len(containers) == 0 {
return nil
}
// Get all container alerts for this system
alertRecords, err := am.hub.FindAllRecords("container_alerts",
dbx.NewExp("system={:system}", dbx.Params{"system": systemRecord.Id}),
)
if err != nil || len(alertRecords) == 0 {
return nil
}
// Group alerts by container
containerAlerts := make(map[string][]*core.Record)
for _, alertRecord := range alertRecords {
containerId := alertRecord.GetString("container")
containerAlerts[containerId] = append(containerAlerts[containerId], alertRecord)
}
// Process alerts for each container
for containerId, alerts := range containerAlerts {
// Find the container in the current stats
var containerStats *container.Stats
for _, c := range containers {
if c.Id == containerId {
containerStats = c
break
}
}
// If container not found in current stats, it might be stopped
// We'll still process Status and Health alerts
if containerStats == nil {
for _, alertRecord := range alerts {
name := alertRecord.GetString("name")
if name == "Status" || name == "Health" {
// Container is missing, so it's likely stopped
go am.sendContainerStatusAlert(alertRecord, systemRecord, containerId, "stopped")
}
}
continue
}
// Process each alert for this container
var validAlerts []ContainerAlertData
now := time.Now().UTC()
oldestTime := now
for _, alertRecord := range alerts {
name := alertRecord.GetString("name")
var val float64
unit := "%"
switch name {
case "CPU":
val = containerStats.Cpu
case "Memory":
val = containerStats.Mem
case "Network":
val = containerStats.NetworkSent + containerStats.NetworkRecv
unit = " MB/s"
case "Status":
// Status alert is handled differently - immediate trigger
triggered := alertRecord.GetBool("triggered")
isRunning := containerStats.Status == "running"
if (!triggered && !isRunning) || (triggered && isRunning) {
go am.sendContainerStatusAlert(alertRecord, systemRecord, containerId, containerStats.Status)
}
continue
case "Health":
// Health alert - trigger on unhealthy status
triggered := alertRecord.GetBool("triggered")
isHealthy := containerStats.Health == container.DockerHealthNone || containerStats.Health == container.DockerHealthHealthy
if (!triggered && !isHealthy) || (triggered && isHealthy) {
go am.sendContainerHealthAlert(alertRecord, systemRecord, containerId, containerStats.Health)
}
continue
default:
continue
}
triggered := alertRecord.GetBool("triggered")
threshold := alertRecord.GetFloat("value")
min := max(1, cast.ToUint8(alertRecord.Get("min")))
alert := ContainerAlertData{
systemRecord: systemRecord,
alertRecord: alertRecord,
name: name,
unit: unit,
val: val,
threshold: threshold,
triggered: triggered,
min: min,
}
// Send alert immediately if min is 1
if min == 1 {
alert.triggered = val > threshold
go am.sendContainerAlert(alert, containerId, containerStats.Name)
continue
}
alert.time = now.Add(-time.Duration(min) * time.Minute)
if alert.time.Before(oldestTime) {
oldestTime = alert.time
}
validAlerts = append(validAlerts, alert)
}
if len(validAlerts) == 0 {
continue
}
// Fetch historical container stats
containerStatsRecords := []struct {
Stats []byte `db:"stats"`
Created types.DateTime `db:"created"`
}{}
err = am.hub.DB().
Select("stats", "created").
From("container_stats").
Where(dbx.NewExp(
"system={:system} AND type='1m' AND created > {:created}",
dbx.Params{
"system": systemRecord.Id,
"created": oldestTime.Add(-time.Second * 90),
},
)).
OrderBy("created").
All(&containerStatsRecords)
if err != nil || len(containerStatsRecords) == 0 {
continue
}
oldestRecordTime := containerStatsRecords[0].Created.Time()
// Filter valid alerts
filteredAlerts := make([]ContainerAlertData, 0, len(validAlerts))
for _, alert := range validAlerts {
if alert.time.After(oldestRecordTime) {
filteredAlerts = append(filteredAlerts, alert)
}
}
validAlerts = filteredAlerts
if len(validAlerts) == 0 {
continue
}
// Process historical stats
for i := range containerStatsRecords {
stat := containerStatsRecords[i]
systemStatsCreation := stat.Created.Time().Add(-time.Second * 10)
// Parse container stats array
var allContainerStats []*container.Stats
if err := json.Unmarshal(stat.Stats, &allContainerStats); err != nil {
continue
}
// Find this container in the stats
var thisContainerStats *container.Stats
for _, cs := range allContainerStats {
if cs.Id == containerId {
thisContainerStats = cs
break
}
}
if thisContainerStats == nil {
continue
}
for j := range validAlerts {
alert := &validAlerts[j]
if i == 0 {
alert.val = 0
}
if systemStatsCreation.Before(alert.time) {
continue
}
switch alert.name {
case "CPU":
alert.val += thisContainerStats.Cpu
case "Memory":
alert.val += thisContainerStats.Mem
case "Network":
alert.val += thisContainerStats.NetworkSent + thisContainerStats.NetworkRecv
default:
continue
}
alert.count++
}
}
// Calculate averages and send alerts
for _, alert := range validAlerts {
alert.val = alert.val / float64(alert.count)
minCount := float32(alert.min) / 1.2
if float32(alert.count) >= minCount {
if !alert.triggered && alert.val > alert.threshold {
alert.triggered = true
go am.sendContainerAlert(alert, containerId, containerStats.Name)
} else if alert.triggered && alert.val <= alert.threshold {
alert.triggered = false
go am.sendContainerAlert(alert, containerId, containerStats.Name)
}
}
}
}
return nil
}
// sendContainerAlert sends a container alert notification
func (am *AlertManager) sendContainerAlert(alert ContainerAlertData, containerId, containerName string) {
systemName := alert.systemRecord.GetString("name")
var subject string
if alert.triggered {
subject = fmt.Sprintf("%s container %s %s above threshold", systemName, containerName, strings.ToLower(alert.name))
} else {
subject = fmt.Sprintf("%s container %s %s below threshold", systemName, containerName, strings.ToLower(alert.name))
}
minutesLabel := "minute"
if alert.min > 1 {
minutesLabel += "s"
}
descriptor := alert.name
if alert.descriptor != "" {
descriptor = alert.descriptor
}
body := fmt.Sprintf("%s averaged %.2f%s for the previous %v %s.", descriptor, alert.val, alert.unit, alert.min, minutesLabel)
alert.alertRecord.Set("triggered", alert.triggered)
if err := am.hub.Save(alert.alertRecord); err != nil {
return
}
am.SendAlert(AlertMessageData{
UserID: alert.alertRecord.GetString("user"),
SystemID: alert.systemRecord.Id,
Title: subject,
Message: body,
Link: am.hub.MakeLink("system", alert.systemRecord.Id, "containers", containerId),
LinkText: "View " + containerName,
})
}
// sendContainerStatusAlert sends a container status change alert
func (am *AlertManager) sendContainerStatusAlert(alertRecord, systemRecord *core.Record, containerId, status string) {
triggered := alertRecord.GetBool("triggered")
isRunning := status == "running"
// Only send if state changed
if (!triggered && isRunning) || (triggered && !isRunning) {
return
}
systemName := systemRecord.GetString("name")
// Try to get container name from containers collection
containerName := containerId
if containerRecord, err := am.hub.FindFirstRecordByFilter("containers", "id={:id}", dbx.Params{"id": containerId}); err == nil {
containerName = containerRecord.GetString("name")
}
var subject string
if isRunning {
subject = fmt.Sprintf("%s container %s is now running", systemName, containerName)
} else {
subject = fmt.Sprintf("%s container %s has stopped", systemName, containerName)
}
body := fmt.Sprintf("Container status changed to: %s", status)
alertRecord.Set("triggered", !isRunning)
if err := am.hub.Save(alertRecord); err != nil {
return
}
am.SendAlert(AlertMessageData{
UserID: alertRecord.GetString("user"),
SystemID: systemRecord.Id,
Title: subject,
Message: body,
Link: am.hub.MakeLink("system", systemRecord.Id, "containers", containerId),
LinkText: "View " + containerName,
})
}
// sendContainerHealthAlert sends a container health status change alert
func (am *AlertManager) sendContainerHealthAlert(alertRecord, systemRecord *core.Record, containerId string, health container.DockerHealth) {
triggered := alertRecord.GetBool("triggered")
isHealthy := health == container.DockerHealthNone || health == container.DockerHealthHealthy
// Only send if state changed
if (!triggered && isHealthy) || (triggered && !isHealthy) {
return
}
systemName := systemRecord.GetString("name")
// Try to get container name from containers collection
containerName := containerId
if containerRecord, err := am.hub.FindFirstRecordByFilter("containers", "id={:id}", dbx.Params{"id": containerId}); err == nil {
containerName = containerRecord.GetString("name")
}
var subject string
if isHealthy {
subject = fmt.Sprintf("%s container %s is now healthy", systemName, containerName)
} else {
subject = fmt.Sprintf("%s container %s is unhealthy", systemName, containerName)
}
// Convert DockerHealth to string
healthStatus := "unknown"
for str, val := range container.DockerHealthStrings {
if val == health {
healthStatus = str
break
}
}
body := fmt.Sprintf("Container health status changed to: %s", healthStatus)
alertRecord.Set("triggered", !isHealthy)
if err := am.hub.Save(alertRecord); err != nil {
return
}
am.SendAlert(AlertMessageData{
UserID: alertRecord.GetString("user"),
SystemID: systemRecord.Id,
Title: subject,
Message: body,
Link: am.hub.MakeLink("system", systemRecord.Id, "containers", containerId),
LinkText: "View " + containerName,
})
}
+67
View File
@@ -72,3 +72,70 @@ func createAlertHistoryRecord(app core.App, alertRecord *core.Record) (alertHist
}
return alertHistoryRecord, err
}
// On triggered container alert record delete, set matching alert history record to resolved
func resolveHistoryOnContainerAlertDelete(e *core.RecordEvent) error {
if !e.Record.GetBool("triggered") {
return e.Next()
}
_ = resolveContainerAlertHistoryRecord(e.App, e.Record.Id)
return e.Next()
}
// On container alert record update, update alert history record
func updateHistoryOnContainerAlertUpdate(e *core.RecordEvent) error {
original := e.Record.Original()
new := e.Record
originalTriggered := original.GetBool("triggered")
newTriggered := new.GetBool("triggered")
// no need to update alert history if triggered state has not changed
if originalTriggered == newTriggered {
return e.Next()
}
// if new state is triggered, create new alert history record
if newTriggered {
_, _ = createContainerAlertHistoryRecord(e.App, new)
return e.Next()
}
// if new state is not triggered, check for matching alert history record and set it to resolved
_ = resolveContainerAlertHistoryRecord(e.App, new.Id)
return e.Next()
}
// resolveContainerAlertHistoryRecord sets the resolved field to the current time
func resolveContainerAlertHistoryRecord(app core.App, alertRecordID string) error {
alertHistoryRecord, err := app.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", dbx.Params{"alert_id": alertRecordID})
if err != nil || alertHistoryRecord == nil {
return err
}
alertHistoryRecord.Set("resolved", time.Now().UTC())
err = app.Save(alertHistoryRecord)
if err != nil {
app.Logger().Error("Failed to resolve container alert history", "err", err)
}
return err
}
// createContainerAlertHistoryRecord creates a new container alert history record
func createContainerAlertHistoryRecord(app core.App, alertRecord *core.Record) (alertHistoryRecord *core.Record, err error) {
alertHistoryCollection, err := app.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return nil, err
}
alertHistoryRecord = core.NewRecord(alertHistoryCollection)
alertHistoryRecord.Set("alert_id", alertRecord.Id)
alertHistoryRecord.Set("user", alertRecord.GetString("user"))
alertHistoryRecord.Set("system", alertRecord.GetString("system"))
alertHistoryRecord.Set("container", alertRecord.GetString("container"))
alertHistoryRecord.Set("name", alertRecord.GetString("name"))
alertHistoryRecord.Set("value", alertRecord.GetFloat("value"))
err = app.Save(alertHistoryRecord)
if err != nil {
app.Logger().Error("Failed to save container alert history", "err", err)
}
return alertHistoryRecord, err
}
+3
View File
@@ -296,6 +296,9 @@ func (h *Hub) registerApiRoutes(se *core.ServeEvent) error {
// update / delete user alerts
apiAuth.POST("/user-alerts", alerts.UpsertUserAlerts)
apiAuth.DELETE("/user-alerts", alerts.DeleteUserAlerts)
// update / delete container alerts
apiAuth.POST("/user-container-alerts", alerts.UpsertUserContainerAlerts)
apiAuth.DELETE("/user-container-alerts", alerts.DeleteUserContainerAlerts)
// refresh SMART devices for a system
apiAuth.POST("/smart/refresh", h.refreshSmartData)
// get systemd service details
+5
View File
@@ -51,6 +51,7 @@ type hubLike interface {
core.App
GetSSHKey(dataDir string) (ssh.Signer, error)
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData) error
HandleStatusAlerts(status string, systemRecord *core.Record) error
}
@@ -211,6 +212,10 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
if err := sm.hub.HandleSystemAlerts(e.Record, system.data); err != nil {
e.App.Logger().Error("Error handling system alerts", "err", err)
}
// Trigger container alerts
if err := sm.hub.HandleContainerAlerts(e.Record, system.data); err != nil {
e.App.Logger().Error("Error handling container alerts", "err", err)
}
}
// Trigger status change alerts for up/down transitions
@@ -0,0 +1,208 @@
package migrations
import (
"encoding/json"
"github.com/pocketbase/pocketbase/core"
m "github.com/pocketbase/pocketbase/migrations"
)
func init() {
m.Register(func(app core.App) error {
// 1. Create container_alerts collection
// Check if collection already exists to prevent errors on restart
if _, err := app.FindCollectionByNameOrId("container_alerts"); err == nil {
// Collection exists, skip creation
} else {
jsonData := `{
"id": "ca_container_alerts",
"name": "container_alerts",
"type": "base",
"system": false,
"listRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
"viewRule": "",
"createRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
"updateRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
"deleteRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
"fields": [
{
"id": "text_id",
"name": "id",
"type": "text",
"required": true,
"presentable": false,
"unique": false,
"system": true,
"primaryKey": true,
"autogeneratePattern": "[a-z0-9]{15}",
"min": 15,
"max": 15,
"pattern": "^[a-z0-9]+$"
},
{
"id": "rel_user",
"name": "user",
"type": "relation",
"required": true,
"presentable": false,
"unique": false,
"system": false,
"collectionId": "_pb_users_auth_",
"cascadeDelete": true,
"minSelect": null,
"maxSelect": 1,
"displayFields": null
},
{
"id": "rel_system",
"name": "system",
"type": "relation",
"required": true,
"presentable": false,
"unique": false,
"system": false,
"collectionId": "2hz5ncl8tizk5nx",
"cascadeDelete": true,
"minSelect": null,
"maxSelect": 1,
"displayFields": null
},
{
"id": "text_container",
"name": "container",
"type": "text",
"required": true,
"presentable": false,
"unique": false,
"system": false,
"min": null,
"max": null,
"pattern": ""
},
{
"id": "select_name",
"name": "name",
"type": "select",
"required": true,
"presentable": false,
"unique": false,
"system": false,
"maxSelect": 1,
"values": [
"Status",
"CPU",
"Memory",
"Network",
"Health"
]
},
{
"id": "number_value",
"name": "value",
"type": "number",
"required": false,
"presentable": false,
"unique": false,
"system": false,
"min": null,
"max": null,
"onlyInt": false
},
{
"id": "number_min",
"name": "min",
"type": "number",
"required": false,
"presentable": false,
"unique": false,
"system": false,
"min": null,
"max": 60,
"onlyInt": true
},
{
"id": "bool_triggered",
"name": "triggered",
"type": "bool",
"required": false,
"presentable": false,
"unique": false,
"system": false
},
{
"id": "autodate_created",
"name": "created",
"type": "autodate",
"required": false,
"presentable": false,
"unique": false,
"system": false,
"onCreate": true,
"onUpdate": false
},
{
"id": "autodate_updated",
"name": "updated",
"type": "autodate",
"required": false,
"presentable": false,
"unique": false,
"system": false,
"onCreate": true,
"onUpdate": true
}
],
"indexes": [
"CREATE UNIQUE INDEX ` + "`" + `idx_container_alerts_unique` + "`" + ` ON ` + "`" + `container_alerts` + "`" + ` (` + "`" + `user` + "`" + `, ` + "`" + `system` + "`" + `, ` + "`" + `container` + "`" + `, ` + "`" + `name` + "`" + `)"
]
}`
collection := &core.Collection{}
if err := json.Unmarshal([]byte(jsonData), collection); err != nil {
return err
}
if err := app.Save(collection); err != nil {
return err
}
}
// 2. Add container field to alerts_history collection
alertsHistoryCollection, err := app.FindCollectionByNameOrId("alerts_history")
if err != nil {
return err
}
if alertsHistoryCollection.Fields.GetByName("container") == nil {
containerField := &core.TextField{
Name: "container",
Required: false,
}
alertsHistoryCollection.Fields.Add(containerField)
return app.Save(alertsHistoryCollection)
}
return nil
}, func(app core.App) error {
// Rollback 2: remove the container field
alertsHistoryCollection, err := app.FindCollectionByNameOrId("alerts_history")
if err == nil {
if field := alertsHistoryCollection.Fields.GetByName("container"); field != nil {
alertsHistoryCollection.Fields.RemoveByName("container")
if err := app.Save(alertsHistoryCollection); err != nil {
return err
}
}
}
// Rollback 1: delete the container_alerts collection
collection, err := app.FindCollectionByNameOrId("container_alerts")
if err == nil {
return app.Delete(collection)
}
return nil
})
}
@@ -0,0 +1,44 @@
import { t } from "@lingui/core/macro"
import { useStore } from "@nanostores/react"
import { BellIcon } from "lucide-react"
import { memo, useMemo, useState } from "react"
import { Button } from "@/components/ui/button"
import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"
import { $containerAlerts } from "@/lib/stores"
import { cn } from "@/lib/utils"
import type { ContainerRecord } from "@/types"
import { ContainerAlertDialogContent } from "./container-alerts-sheet"
export default memo(function ContainerAlertButton({
systemId,
container,
}: {
systemId: string
container: ContainerRecord
}) {
const [opened, setOpened] = useState(false)
const alerts = useStore($containerAlerts)
const containerAlerts = alerts[systemId]?.get(container.id)
const hasContainerAlert = containerAlerts && containerAlerts.size > 0
return useMemo(
() => (
<Sheet>
<SheetTrigger asChild>
<Button variant="ghost" size="icon" aria-label={t`Alerts`} data-nolink onClick={() => setOpened(true)}>
<BellIcon
className={cn("h-[1.2em] w-[1.2em] pointer-events-none", {
"fill-primary": hasContainerAlert,
})}
/>
</Button>
</SheetTrigger>
<SheetContent className="max-h-full overflow-auto w-150 !max-w-full p-4 sm:p-6">
{opened && <ContainerAlertDialogContent systemId={systemId} container={container} />}
</SheetContent>
</Sheet>
),
[opened, hasContainerAlert]
)
})
@@ -0,0 +1,327 @@
import { t } from "@lingui/core/macro"
import { Plural, Trans } from "@lingui/react/macro"
import { useStore } from "@nanostores/react"
import { getPagePath } from "@nanostores/router"
import { BoxIcon, GlobeIcon } from "lucide-react"
import { lazy, memo, Suspense, useMemo, useState } from "react"
import { $router, Link } from "@/components/router"
import { Checkbox } from "@/components/ui/checkbox"
import { DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"
import { Switch } from "@/components/ui/switch"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import { toast } from "@/components/ui/use-toast"
import { containerAlertInfo } from "@/lib/container-alerts"
import { pb } from "@/lib/api"
import { $containerAlerts } from "@/lib/stores"
import { cn, debounce } from "@/lib/utils"
import type { ContainerAlertInfo, ContainerAlertRecord, ContainerRecord } from "@/types"
const Slider = lazy(() => import("@/components/ui/slider"))
const endpoint = "/api/beszel/user-container-alerts"
const alertDebounce = 100
const alertKeys = Object.keys(containerAlertInfo) as (keyof typeof containerAlertInfo)[]
const failedUpdateToast = (error: unknown) => {
console.error(error)
toast({
title: t`Failed to update alert`,
description: t`Please check logs for more details.`,
variant: "destructive",
})
}
/** Create or update container alerts */
const upsertContainerAlerts = debounce(
async ({
name,
value,
min,
systems,
containers,
}: {
name: string
value: number
min: number
systems: string[]
containers: string[]
}) => {
try {
await pb.send<{ success: boolean }>(endpoint, {
method: "POST",
body: { name, value, min, systems, containers, overwrite: true },
})
} catch (error) {
failedUpdateToast(error)
}
},
alertDebounce
)
/** Delete container alerts */
const deleteContainerAlerts = debounce(
async ({ name, systems, containers }: { name: string; systems: string[]; containers: string[] }) => {
try {
await pb.send<{ success: boolean }>(endpoint, {
method: "DELETE",
body: { name, systems, containers },
})
} catch (error) {
failedUpdateToast(error)
}
},
alertDebounce
)
export const ContainerAlertDialogContent = memo(function ContainerAlertDialogContent({
systemId,
container,
}: {
systemId: string
container: ContainerRecord
}) {
const alerts = useStore($containerAlerts)
const [overwriteExisting, setOverwriteExisting] = useState<boolean | "indeterminate">(false)
const [currentTab, setCurrentTab] = useState("container")
const containerAlerts = alerts[systemId]?.get(container.id) ?? new Map()
// Keep a copy of alerts when we switch to global tab
const alertsWhenGlobalSelected = useMemo(() => {
return currentTab === "global" ? structuredClone(alerts) : alerts
}, [currentTab])
return (
<>
<DialogHeader>
<DialogTitle className="text-xl">
<Trans>Container Alerts</Trans>
</DialogTitle>
<DialogDescription>
<Trans>
See{" "}
<Link href={getPagePath($router, "settings", { name: "notifications" })} className="link">
notification settings
</Link>{" "}
to configure how you receive alerts.
</Trans>
</DialogDescription>
</DialogHeader>
<Tabs defaultValue="container" onValueChange={setCurrentTab}>
<TabsList className="mb-1 -mt-0.5">
<TabsTrigger value="container">
<BoxIcon className="me-2 h-3.5 w-3.5" />
<span className="truncate max-w-60">{container.name}</span>
</TabsTrigger>
<TabsTrigger value="global">
<GlobeIcon className="me-1.5 h-3.5 w-3.5" />
<Trans>All Containers</Trans>
</TabsTrigger>
</TabsList>
<TabsContent value="container">
<div className="grid gap-3">
{alertKeys.map((name) => (
<ContainerAlertContent
key={name}
alertKey={name}
data={containerAlertInfo[name as keyof typeof containerAlertInfo]}
alert={containerAlerts.get(name)}
systemId={systemId}
container={container}
/>
))}
</div>
</TabsContent>
<TabsContent value="global">
<label
htmlFor="ovw"
className="mb-3 flex gap-2 items-center justify-center cursor-pointer border rounded-sm py-3 px-4 border-destructive text-destructive font-semibold text-sm"
>
<Checkbox
id="ovw"
className="text-destructive border-destructive data-[state=checked]:bg-destructive"
checked={overwriteExisting}
onCheckedChange={setOverwriteExisting}
/>
<Trans>Overwrite existing alerts</Trans>
</label>
<div className="grid gap-3">
{alertKeys.map((name) => (
<ContainerAlertContent
key={name}
alertKey={name}
systemId={systemId}
container={container}
alert={containerAlerts.get(name)}
data={containerAlertInfo[name as keyof typeof containerAlertInfo]}
global={true}
overwriteExisting={!!overwriteExisting}
initialAlertsState={alertsWhenGlobalSelected}
/>
))}
</div>
</TabsContent>
</Tabs>
</>
)
})
export function ContainerAlertContent({
alertKey,
data: alertData,
systemId,
container,
alert,
global = false,
overwriteExisting = false,
initialAlertsState = {},
}: {
alertKey: string
data: ContainerAlertInfo
systemId: string
container: ContainerRecord
alert?: ContainerAlertRecord
global?: boolean
overwriteExisting?: boolean
initialAlertsState?: Record<string, Map<string, Map<string, ContainerAlertRecord>>>
}) {
const { name } = alertData
const singleDescription = alertData.singleDesc?.()
const [checked, setChecked] = useState(global ? false : !!alert)
const [min, setMin] = useState(alert?.min || 10)
const [value, setValue] = useState(alert?.value || (singleDescription ? 0 : alertData.start ?? 80))
const Icon = alertData.icon
/** Get container ids to update */
function getContainerIds(): string[] {
// if not global, update only the current container
if (!global) {
return [container.id]
}
// if global, we need to get all containers for this system
// For now, we'll just use the current container
// In a real implementation, you'd fetch all containers for the system
return [container.id]
}
function sendUpsert(min: number, value: number) {
const containers = getContainerIds()
containers.length &&
upsertContainerAlerts({
name: alertKey,
value,
min,
systems: [systemId],
containers,
})
}
return (
<div className="rounded-lg border border-muted-foreground/15 hover:border-muted-foreground/20 transition-colors duration-100 group">
<label
htmlFor={`c${name}`}
className={cn("flex flex-row items-center justify-between gap-4 cursor-pointer p-4", {
"pb-0": checked,
})}
>
<div className="grid gap-1 select-none">
<p className="font-semibold flex gap-3 items-center">
<Icon className="h-4 w-4 opacity-85" /> {alertData.name()}
</p>
{!checked && <span className="block text-sm text-muted-foreground">{alertData.desc()}</span>}
</div>
<Switch
id={`c${name}`}
checked={checked}
onCheckedChange={(newChecked) => {
setChecked(newChecked)
if (newChecked) {
// if alert checked, create or update alert
sendUpsert(min, value)
} else {
// if unchecked, delete alert
deleteContainerAlerts({ name: alertKey, systems: [systemId], containers: getContainerIds() })
// when force deleting all alerts of a type, also remove them from initialAlertsState
if (overwriteExisting) {
for (const systemAlerts of Object.values(initialAlertsState)) {
for (const containerAlerts of systemAlerts.values()) {
containerAlerts.delete(alertKey)
}
}
}
}
}}
/>
</label>
{checked && (
<div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground">
<Suspense fallback={<div className="h-10" />}>
{!singleDescription && (
<div>
<p id={`v${name}`} className="text-sm block h-8">
{alertData.invert ? (
<Trans>
Average drops below{" "}
<strong className="text-foreground">
{value}
{alertData.unit}
</strong>
</Trans>
) : (
<Trans>
Average exceeds{" "}
<strong className="text-foreground">
{value}
{alertData.unit}
</strong>
</Trans>
)}
</p>
<div className="flex gap-3">
<Slider
aria-labelledby={`v${name}`}
defaultValue={[value]}
onValueCommit={(val) => sendUpsert(min, val[0])}
onValueChange={(val) => setValue(val[0])}
step={alertData.step ?? 1}
min={alertData.min ?? 1}
max={alertData.max ?? 99}
/>
</div>
</div>
)}
<div className={cn(singleDescription && "col-span-full lowercase")}>
<p id={`t${name}`} className="text-sm block h-8 first-letter:uppercase">
{singleDescription && (
<>
{singleDescription}
{` `}
</>
)}
<Trans>
For <strong className="text-foreground">{min}</strong>{" "}
<Plural value={min} one="minute" other="minutes" />
</Trans>
</p>
<div className="flex gap-3">
<Slider
aria-labelledby={`v${name}`}
defaultValue={[min]}
onValueCommit={(minVal) => sendUpsert(minVal[0], value)}
onValueChange={(val) => setMin(val[0])}
min={1}
max={60}
/>
</div>
</div>
</Suspense>
</div>
)}
</div>
)
}
@@ -1,4 +1,5 @@
import type { Column, ColumnDef } from "@tanstack/react-table"
import { lazy, Suspense } from "react"
import { Button } from "@/components/ui/button"
import { cn, decimalString, formatBytes, hourWithSeconds } from "@/lib/utils"
import type { ContainerRecord } from "@/types"
@@ -40,6 +41,9 @@ function getStatusValue(status: string): number {
return 0
}
// Lazy load the alert button component
const ContainerAlertButton = lazy(() => import("@/components/alerts/container-alert-button"))
export const containerChartCols: ColumnDef<ContainerRecord>[] = [
{
id: "name",
@@ -62,7 +66,7 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
cell: ({ getValue }) => {
const allSystems = useStore($allSystemsById)
return <span className="ms-1.5 xl:w-34 block truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
return <span className="ms-1.5 xl:w-30 block truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
},
},
// {
@@ -137,17 +141,17 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
id: "image",
sortingFn: (a, b) => a.original.image.localeCompare(b.original.image),
accessorFn: (record) => record.image,
header: ({ column }) => (
<HeaderButton column={column} name={t({ message: "Image", context: "Docker image" })} Icon={LayersIcon} />
),
enableHiding: true,
header: ({ column }) => <HeaderButton column={column} name={t({ message: "Image", context: "Docker image" })} Icon={LayersIcon} />,
cell: ({ getValue }) => {
return <span className="ms-1.5 xl:w-40 block truncate">{getValue() as string}</span>
return <span className="ms-1.5 xl:w-30 block truncate">{getValue() as string}</span>
},
},
{
id: "status",
accessorFn: (record) => record.status,
invertSorting: true,
enableHiding: true,
sortingFn: (a, b) => getStatusValue(a.original.status) - getStatusValue(b.original.status),
header: ({ column }) => <HeaderButton column={column} name={t`Status`} Icon={HourglassIcon} />,
cell: ({ getValue }) => {
@@ -164,6 +168,21 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
return <span className="ms-1.5 tabular-nums">{hourWithSeconds(new Date(timestamp).toISOString())}</span>
},
},
{
id: "actions",
header: () => <div className="text-center">{t`Actions`}</div>,
cell: ({ row }) => {
// Lazy load the alert button component
const ContainerAlertButton = lazy(() => import("@/components/alerts/container-alert-button"))
return (
<div className="flex justify-center" onClick={(e) => e.stopPropagation()}>
<Suspense fallback={<div className="h-8 w-8" />}>
<ContainerAlertButton systemId={row.original.system} container={row.original} />
</Suspense>
</div>
)
},
},
]
function HeaderButton({
@@ -231,7 +231,7 @@ const AllContainersTable = memo(function AllContainersTable({
>
{/* add header height to table size */}
<div style={{ height: `${virtualizer.getTotalSize() + 48}px`, paddingTop, paddingBottom }}>
<table className="text-sm w-full h-full text-nowrap">
<table className="text-sm w-full h-full">
<ContainersTableHead table={table} />
<TableBody>
{rows.length ? (
+2 -1
View File
@@ -3,7 +3,7 @@ import PocketBase from "pocketbase"
import { basePath } from "@/components/router"
import { toast } from "@/components/ui/use-toast"
import type { ChartTimes, UserSettings } from "@/types"
import { $alerts, $allSystemsById, $allSystemsByName, $userSettings } from "./stores"
import { $alerts, $allSystemsById, $allSystemsByName, $containerAlerts, $userSettings } from "./stores"
import { chartTimeData } from "./utils"
/** PocketBase JS Client */
@@ -30,6 +30,7 @@ export function logOut() {
$allSystemsByName.set({})
$allSystemsById.set({})
$alerts.set({})
$containerAlerts.set({})
$userSettings.set({} as UserSettings)
sessionStorage.setItem("lo", "t") // prevent auto login on logout
pb.authStore.clear()
+180
View File
@@ -0,0 +1,180 @@
import { t } from "@lingui/core/macro"
import { CpuIcon, HeartPulseIcon, MemoryStickIcon, ServerIcon } from "lucide-react"
import type { RecordSubscription } from "pocketbase"
import { EthernetIcon } from "@/components/ui/icons"
import { $containerAlerts } from "@/lib/stores"
import type { ContainerAlertInfo, ContainerAlertRecord } from "@/types"
import { pb } from "./api"
/** Container alert info for each alert type */
export const containerAlertInfo: Record<string, ContainerAlertInfo> = {
Status: {
name: () => t`Status`,
unit: "",
icon: ServerIcon,
desc: () => t`Triggers when container status changes`,
singleDesc: () => `${t`Container`} ${t`Stopped`}`,
},
CPU: {
name: () => t`CPU Usage`,
unit: "%",
icon: CpuIcon,
desc: () => t`Triggers when CPU usage exceeds a threshold`,
},
Memory: {
name: () => t`Memory Usage`,
unit: "%",
icon: MemoryStickIcon,
desc: () => t`Triggers when memory usage exceeds a threshold`,
},
Network: {
name: () => t`Network`,
unit: " MB/s",
icon: EthernetIcon,
desc: () => t`Triggers when combined up/down exceeds a threshold`,
max: 125,
},
Health: {
name: () => t`Health Status`,
unit: "",
icon: HeartPulseIcon,
desc: () => t`Triggers when container health status changes`,
singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
},
}
class ContainerAlertManager {
private unsubscribeFn?: () => void
/**
* Subscribe to container alert updates
*/
async subscribe() {
if (this.unsubscribeFn) {
return
}
// Fetch initial container alerts
try {
const alerts = await pb.collection("container_alerts").getFullList<ContainerAlertRecord>()
this.updateStore(alerts)
} catch (e) {
console.error("Failed to fetch container alerts:", e)
}
// Subscribe to real-time updates
this.unsubscribeFn = await pb
.collection("container_alerts")
.subscribe<ContainerAlertRecord>("*", this.handleAlertUpdate)
}
/**
* Unsubscribe from container alert updates
*/
unsubscribe() {
if (this.unsubscribeFn) {
this.unsubscribeFn()
this.unsubscribeFn = undefined
}
}
/**
* Handle real-time alert updates
*/
private handleAlertUpdate = (e: RecordSubscription<ContainerAlertRecord>) => {
const { action, record } = e
if (action === "delete") {
this.deleteFromStore(record)
} else {
this.updateStore([record])
}
}
/**
* Update store with alert records
*/
private updateStore(alerts: ContainerAlertRecord[]) {
const currentAlerts = $containerAlerts.get()
for (const alert of alerts) {
if (!currentAlerts[alert.system]) {
currentAlerts[alert.system] = new Map()
}
if (!currentAlerts[alert.system].get(alert.container)) {
currentAlerts[alert.system].set(alert.container, new Map())
}
currentAlerts[alert.system].get(alert.container)!.set(alert.name, alert)
}
$containerAlerts.set(currentAlerts)
}
/**
* Delete alert from store
*/
private deleteFromStore(alert: ContainerAlertRecord) {
const currentAlerts = $containerAlerts.get()
if (currentAlerts[alert.system]?.get(alert.container)?.has(alert.name)) {
currentAlerts[alert.system].get(alert.container)!.delete(alert.name)
// Clean up empty maps
if (currentAlerts[alert.system].get(alert.container)!.size === 0) {
currentAlerts[alert.system].delete(alert.container)
}
if (currentAlerts[alert.system].size === 0) {
delete currentAlerts[alert.system]
}
$containerAlerts.set(currentAlerts)
}
}
/**
* Create or update container alerts
*/
async upsert(
systems: string[],
containers: string[],
name: string,
value: number,
min: number,
overwrite = false
) {
return pb.send("/api/beszel/user-container-alerts", {
method: "POST",
body: {
systems,
containers,
name,
value,
min,
overwrite,
},
})
}
/**
* Delete container alerts
*/
async delete(systems: string[], containers: string[], name: string) {
return pb.send("/api/beszel/user-container-alerts", {
method: "DELETE",
body: {
systems,
containers,
name,
},
})
}
/**
* Clear all container alerts from store
*/
clear() {
$containerAlerts.set({})
}
}
export const containerAlertManager = new ContainerAlertManager()
+4 -1
View File
@@ -1,5 +1,5 @@
import { atom, computed, listenKeys, map, type ReadableAtom } from "nanostores"
import type { AlertMap, ChartTimes, SystemRecord, UserSettings } from "@/types"
import type { AlertMap, ChartTimes, ContainerAlertMap, SystemRecord, UserSettings } from "@/types"
import { pb } from "./api"
import { Unit } from "./enums"
@@ -22,6 +22,9 @@ export const $systems: ReadableAtom<SystemRecord[]> = computed($allSystemsById,
/** Map of alert records by system id and alert name */
export const $alerts = map<AlertMap>({})
/** Map of container alert records by system id, container id, and alert name */
export const $containerAlerts = map<ContainerAlertMap>({})
/** SSH public key */
export const $publicKey = atom("")
+4
View File
@@ -12,6 +12,7 @@ import Settings from "@/components/routes/settings/layout.tsx"
import { ThemeProvider } from "@/components/theme-provider.tsx"
import { Toaster } from "@/components/ui/toaster.tsx"
import { alertManager } from "@/lib/alerts"
import { containerAlertManager } from "@/lib/container-alerts"
import { pb, updateUserSettings } from "@/lib/api.ts"
import { dynamicActivate, getLocale } from "@/lib/i18n"
import { $authenticated, $copyContent, $direction, $publicKey, $userSettings } from "@/lib/stores.ts"
@@ -49,8 +50,11 @@ const App = memo(() => {
.then(alertManager.refresh)
// subscribe to new alert updates
.then(alertManager.subscribe)
// subscribe to container alerts
.then(() => containerAlertManager.subscribe())
return () => {
alertManager.unsubscribe()
containerAlertManager.unsubscribe()
systemsManager.unsubscribe()
}
}, [])
+27
View File
@@ -340,6 +340,33 @@ export interface AlertInfo {
export type AlertMap = Record<string, Map<string, AlertRecord>>
export interface ContainerAlertRecord extends RecordModel {
id: string
system: string
container: string
name: string
triggered: boolean
value: number
min: number
}
export interface ContainerAlertInfo {
name: () => string
unit: string
icon: any
desc: () => string
max?: number
min?: number
step?: number
start?: number
/** Single value description (when there's only one value, like status) */
singleDesc?: () => string
invert?: boolean
}
export type ContainerAlertMap = Record<string, Map<string, Map<string, ContainerAlertRecord>>>
export interface SmartData {
/** model family */
// mf?: string