diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 3e33bd700..5b49ec45c 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -105,6 +105,8 @@ func NewAlertManager(app hubLike) *AlertManager { func (am *AlertManager) bindEvents() { am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate) am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete) + am.hub.OnRecordAfterUpdateSuccess("container_alerts").BindFunc(updateHistoryOnContainerAlertUpdate) + am.hub.OnRecordAfterDeleteSuccess("container_alerts").BindFunc(resolveHistoryOnContainerAlertDelete) am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert) } diff --git a/internal/alerts/alerts_api.go b/internal/alerts/alerts_api.go index 972f01bf1..a0eb8016d 100644 --- a/internal/alerts/alerts_api.go +++ b/internal/alerts/alerts_api.go @@ -117,3 +117,119 @@ func DeleteUserAlerts(e *core.RequestEvent) error { return e.JSON(http.StatusOK, map[string]any{"success": true, "count": numDeleted}) } + +// UpsertUserContainerAlerts handles API request to create or update container alerts for a user +// across multiple containers (POST /api/beszel/user-container-alerts) +func UpsertUserContainerAlerts(e *core.RequestEvent) error { + userID := e.Auth.Id + + reqData := struct { + Min uint8 `json:"min"` + Value float64 `json:"value"` + Name string `json:"name"` + Systems []string `json:"systems"` + Containers []string `json:"containers"` + Overwrite bool `json:"overwrite"` + }{} + err := e.BindBody(&reqData) + if err != nil || userID == "" || reqData.Name == "" || len(reqData.Systems) == 0 || len(reqData.Containers) == 0 { + return e.BadRequestError("Bad data", err) + } + + containerAlertsCollection, err := e.App.FindCachedCollectionByNameOrId("container_alerts") + if err != nil { + return err + } + + err = e.App.RunInTransaction(func(txApp core.App) error { + for _, systemId := range reqData.Systems { + for _, containerId := range reqData.Containers { + // find existing matching alert + alertRecord, err := txApp.FindFirstRecordByFilter(containerAlertsCollection, + "system={:system} && container={:container} && name={:name} && user={:user}", + dbx.Params{"system": systemId, "container": containerId, "name": reqData.Name, "user": userID}) + + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + + // skip if alert already exists and overwrite is not set + if !reqData.Overwrite && alertRecord != nil { + continue + } + + // create new alert if it doesn't exist + if alertRecord == nil { + alertRecord = core.NewRecord(containerAlertsCollection) + alertRecord.Set("user", userID) + alertRecord.Set("system", systemId) + alertRecord.Set("container", containerId) + alertRecord.Set("name", reqData.Name) + } + + alertRecord.Set("value", reqData.Value) + alertRecord.Set("min", reqData.Min) + + if err := txApp.SaveNoValidate(alertRecord); err != nil { + return err + } + } + } + return nil + }) + + if err != nil { + return err + } + + return e.JSON(http.StatusOK, map[string]any{"success": true}) +} + +// DeleteUserContainerAlerts handles API request to delete container alerts for a user +// across multiple containers (DELETE /api/beszel/user-container-alerts) +func DeleteUserContainerAlerts(e *core.RequestEvent) error { + userID := e.Auth.Id + + reqData := struct { + AlertName string `json:"name"` + Systems []string `json:"systems"` + Containers []string `json:"containers"` + }{} + err := e.BindBody(&reqData) + if err != nil || userID == "" || reqData.AlertName == "" || len(reqData.Systems) == 0 || len(reqData.Containers) == 0 { + return e.BadRequestError("Bad data", err) + } + + var numDeleted uint16 + + err = e.App.RunInTransaction(func(txApp core.App) error { + for _, systemId := range reqData.Systems { + for _, containerId := range reqData.Containers { + // Find existing alert to delete + alertRecord, err := txApp.FindFirstRecordByFilter("container_alerts", + "system={:system} && container={:container} && name={:name} && user={:user}", + dbx.Params{"system": systemId, "container": containerId, "name": reqData.AlertName, "user": userID}) + + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + // alert doesn't exist, continue to next container + continue + } + return err + } + + if err := txApp.Delete(alertRecord); err != nil { + return err + } + numDeleted++ + } + } + return nil + }) + + if err != nil { + return err + } + + return e.JSON(http.StatusOK, map[string]any{"success": true, "count": numDeleted}) +} diff --git a/internal/alerts/alerts_container.go b/internal/alerts/alerts_container.go new file mode 100644 index 000000000..a1650e11a --- /dev/null +++ b/internal/alerts/alerts_container.go @@ -0,0 +1,394 @@ +package alerts + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/henrygd/beszel/internal/entities/container" + "github.com/henrygd/beszel/internal/entities/system" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/types" + "github.com/spf13/cast" +) + +// ContainerAlertData holds the data for a container alert +type ContainerAlertData struct { + systemRecord *core.Record + containerRecord *core.Record + alertRecord *core.Record + name string + unit string + val float64 + threshold float64 + triggered bool + time time.Time + count uint8 + min uint8 + descriptor string // override descriptor in notification body +} + +// ContainerAlertStats represents the stats structure for container alerts +type ContainerAlertStats struct { + Cpu float64 `json:"c"` + Mem float64 `json:"m"` + NetworkSent float64 `json:"ns"` + NetworkRecv float64 `json:"nr"` + Status string `json:"s"` + Health string `json:"h"` +} + +// HandleContainerAlerts processes container alerts for a system +func (am *AlertManager) HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData) error { + containers := data.Containers + if len(containers) == 0 { + return nil + } + + // Get all container alerts for this system + alertRecords, err := am.hub.FindAllRecords("container_alerts", + dbx.NewExp("system={:system}", dbx.Params{"system": systemRecord.Id}), + ) + if err != nil || len(alertRecords) == 0 { + return nil + } + + // Group alerts by container + containerAlerts := make(map[string][]*core.Record) + for _, alertRecord := range alertRecords { + containerId := alertRecord.GetString("container") + containerAlerts[containerId] = append(containerAlerts[containerId], alertRecord) + } + + // Process alerts for each container + for containerId, alerts := range containerAlerts { + // Find the container in the current stats + var containerStats *container.Stats + for _, c := range containers { + if c.Id == containerId { + containerStats = c + break + } + } + + // If container not found in current stats, it might be stopped + // We'll still process Status and Health alerts + if containerStats == nil { + for _, alertRecord := range alerts { + name := alertRecord.GetString("name") + if name == "Status" || name == "Health" { + // Container is missing, so it's likely stopped + go am.sendContainerStatusAlert(alertRecord, systemRecord, containerId, "stopped") + } + } + continue + } + + // Process each alert for this container + var validAlerts []ContainerAlertData + now := time.Now().UTC() + oldestTime := now + + for _, alertRecord := range alerts { + name := alertRecord.GetString("name") + var val float64 + unit := "%" + + switch name { + case "CPU": + val = containerStats.Cpu + case "Memory": + val = containerStats.Mem + case "Network": + val = containerStats.NetworkSent + containerStats.NetworkRecv + unit = " MB/s" + case "Status": + // Status alert is handled differently - immediate trigger + triggered := alertRecord.GetBool("triggered") + isRunning := containerStats.Status == "running" + if (!triggered && !isRunning) || (triggered && isRunning) { + go am.sendContainerStatusAlert(alertRecord, systemRecord, containerId, containerStats.Status) + } + continue + case "Health": + // Health alert - trigger on unhealthy status + triggered := alertRecord.GetBool("triggered") + isHealthy := containerStats.Health == container.DockerHealthNone || containerStats.Health == container.DockerHealthHealthy + if (!triggered && !isHealthy) || (triggered && isHealthy) { + go am.sendContainerHealthAlert(alertRecord, systemRecord, containerId, containerStats.Health) + } + continue + default: + continue + } + + triggered := alertRecord.GetBool("triggered") + threshold := alertRecord.GetFloat("value") + min := max(1, cast.ToUint8(alertRecord.Get("min"))) + + alert := ContainerAlertData{ + systemRecord: systemRecord, + alertRecord: alertRecord, + name: name, + unit: unit, + val: val, + threshold: threshold, + triggered: triggered, + min: min, + } + + // Send alert immediately if min is 1 + if min == 1 { + alert.triggered = val > threshold + go am.sendContainerAlert(alert, containerId, containerStats.Name) + continue + } + + alert.time = now.Add(-time.Duration(min) * time.Minute) + if alert.time.Before(oldestTime) { + oldestTime = alert.time + } + + validAlerts = append(validAlerts, alert) + } + + if len(validAlerts) == 0 { + continue + } + + // Fetch historical container stats + containerStatsRecords := []struct { + Stats []byte `db:"stats"` + Created types.DateTime `db:"created"` + }{} + + err = am.hub.DB(). + Select("stats", "created"). + From("container_stats"). + Where(dbx.NewExp( + "system={:system} AND type='1m' AND created > {:created}", + dbx.Params{ + "system": systemRecord.Id, + "created": oldestTime.Add(-time.Second * 90), + }, + )). + OrderBy("created"). + All(&containerStatsRecords) + + if err != nil || len(containerStatsRecords) == 0 { + continue + } + + oldestRecordTime := containerStatsRecords[0].Created.Time() + + // Filter valid alerts + filteredAlerts := make([]ContainerAlertData, 0, len(validAlerts)) + for _, alert := range validAlerts { + if alert.time.After(oldestRecordTime) { + filteredAlerts = append(filteredAlerts, alert) + } + } + validAlerts = filteredAlerts + + if len(validAlerts) == 0 { + continue + } + + // Process historical stats + for i := range containerStatsRecords { + stat := containerStatsRecords[i] + systemStatsCreation := stat.Created.Time().Add(-time.Second * 10) + + // Parse container stats array + var allContainerStats []*container.Stats + if err := json.Unmarshal(stat.Stats, &allContainerStats); err != nil { + continue + } + + // Find this container in the stats + var thisContainerStats *container.Stats + for _, cs := range allContainerStats { + if cs.Id == containerId { + thisContainerStats = cs + break + } + } + + if thisContainerStats == nil { + continue + } + + for j := range validAlerts { + alert := &validAlerts[j] + if i == 0 { + alert.val = 0 + } + if systemStatsCreation.Before(alert.time) { + continue + } + + switch alert.name { + case "CPU": + alert.val += thisContainerStats.Cpu + case "Memory": + alert.val += thisContainerStats.Mem + case "Network": + alert.val += thisContainerStats.NetworkSent + thisContainerStats.NetworkRecv + default: + continue + } + alert.count++ + } + } + + // Calculate averages and send alerts + for _, alert := range validAlerts { + alert.val = alert.val / float64(alert.count) + minCount := float32(alert.min) / 1.2 + + if float32(alert.count) >= minCount { + if !alert.triggered && alert.val > alert.threshold { + alert.triggered = true + go am.sendContainerAlert(alert, containerId, containerStats.Name) + } else if alert.triggered && alert.val <= alert.threshold { + alert.triggered = false + go am.sendContainerAlert(alert, containerId, containerStats.Name) + } + } + } + } + + return nil +} + +// sendContainerAlert sends a container alert notification +func (am *AlertManager) sendContainerAlert(alert ContainerAlertData, containerId, containerName string) { + systemName := alert.systemRecord.GetString("name") + + var subject string + if alert.triggered { + subject = fmt.Sprintf("%s container %s %s above threshold", systemName, containerName, strings.ToLower(alert.name)) + } else { + subject = fmt.Sprintf("%s container %s %s below threshold", systemName, containerName, strings.ToLower(alert.name)) + } + + minutesLabel := "minute" + if alert.min > 1 { + minutesLabel += "s" + } + + descriptor := alert.name + if alert.descriptor != "" { + descriptor = alert.descriptor + } + + body := fmt.Sprintf("%s averaged %.2f%s for the previous %v %s.", descriptor, alert.val, alert.unit, alert.min, minutesLabel) + + alert.alertRecord.Set("triggered", alert.triggered) + if err := am.hub.Save(alert.alertRecord); err != nil { + return + } + + am.SendAlert(AlertMessageData{ + UserID: alert.alertRecord.GetString("user"), + SystemID: alert.systemRecord.Id, + Title: subject, + Message: body, + Link: am.hub.MakeLink("system", alert.systemRecord.Id, "containers", containerId), + LinkText: "View " + containerName, + }) +} + +// sendContainerStatusAlert sends a container status change alert +func (am *AlertManager) sendContainerStatusAlert(alertRecord, systemRecord *core.Record, containerId, status string) { + triggered := alertRecord.GetBool("triggered") + isRunning := status == "running" + + // Only send if state changed + if (!triggered && isRunning) || (triggered && !isRunning) { + return + } + + systemName := systemRecord.GetString("name") + + // Try to get container name from containers collection + containerName := containerId + if containerRecord, err := am.hub.FindFirstRecordByFilter("containers", "id={:id}", dbx.Params{"id": containerId}); err == nil { + containerName = containerRecord.GetString("name") + } + + var subject string + if isRunning { + subject = fmt.Sprintf("%s container %s is now running", systemName, containerName) + } else { + subject = fmt.Sprintf("%s container %s has stopped", systemName, containerName) + } + + body := fmt.Sprintf("Container status changed to: %s", status) + + alertRecord.Set("triggered", !isRunning) + if err := am.hub.Save(alertRecord); err != nil { + return + } + + am.SendAlert(AlertMessageData{ + UserID: alertRecord.GetString("user"), + SystemID: systemRecord.Id, + Title: subject, + Message: body, + Link: am.hub.MakeLink("system", systemRecord.Id, "containers", containerId), + LinkText: "View " + containerName, + }) +} + +// sendContainerHealthAlert sends a container health status change alert +func (am *AlertManager) sendContainerHealthAlert(alertRecord, systemRecord *core.Record, containerId string, health container.DockerHealth) { + triggered := alertRecord.GetBool("triggered") + isHealthy := health == container.DockerHealthNone || health == container.DockerHealthHealthy + + // Only send if state changed + if (!triggered && isHealthy) || (triggered && !isHealthy) { + return + } + + systemName := systemRecord.GetString("name") + + // Try to get container name from containers collection + containerName := containerId + if containerRecord, err := am.hub.FindFirstRecordByFilter("containers", "id={:id}", dbx.Params{"id": containerId}); err == nil { + containerName = containerRecord.GetString("name") + } + + var subject string + if isHealthy { + subject = fmt.Sprintf("%s container %s is now healthy", systemName, containerName) + } else { + subject = fmt.Sprintf("%s container %s is unhealthy", systemName, containerName) + } + + // Convert DockerHealth to string + healthStatus := "unknown" + for str, val := range container.DockerHealthStrings { + if val == health { + healthStatus = str + break + } + } + body := fmt.Sprintf("Container health status changed to: %s", healthStatus) + + alertRecord.Set("triggered", !isHealthy) + if err := am.hub.Save(alertRecord); err != nil { + return + } + + am.SendAlert(AlertMessageData{ + UserID: alertRecord.GetString("user"), + SystemID: systemRecord.Id, + Title: subject, + Message: body, + Link: am.hub.MakeLink("system", systemRecord.Id, "containers", containerId), + LinkText: "View " + containerName, + }) +} diff --git a/internal/alerts/alerts_history.go b/internal/alerts/alerts_history.go index 9654456fd..4fb6bcd7c 100644 --- a/internal/alerts/alerts_history.go +++ b/internal/alerts/alerts_history.go @@ -72,3 +72,70 @@ func createAlertHistoryRecord(app core.App, alertRecord *core.Record) (alertHist } return alertHistoryRecord, err } + +// On triggered container alert record delete, set matching alert history record to resolved +func resolveHistoryOnContainerAlertDelete(e *core.RecordEvent) error { + if !e.Record.GetBool("triggered") { + return e.Next() + } + _ = resolveContainerAlertHistoryRecord(e.App, e.Record.Id) + return e.Next() +} + +// On container alert record update, update alert history record +func updateHistoryOnContainerAlertUpdate(e *core.RecordEvent) error { + original := e.Record.Original() + new := e.Record + + originalTriggered := original.GetBool("triggered") + newTriggered := new.GetBool("triggered") + + // no need to update alert history if triggered state has not changed + if originalTriggered == newTriggered { + return e.Next() + } + + // if new state is triggered, create new alert history record + if newTriggered { + _, _ = createContainerAlertHistoryRecord(e.App, new) + return e.Next() + } + + // if new state is not triggered, check for matching alert history record and set it to resolved + _ = resolveContainerAlertHistoryRecord(e.App, new.Id) + return e.Next() +} + +// resolveContainerAlertHistoryRecord sets the resolved field to the current time +func resolveContainerAlertHistoryRecord(app core.App, alertRecordID string) error { + alertHistoryRecord, err := app.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", dbx.Params{"alert_id": alertRecordID}) + if err != nil || alertHistoryRecord == nil { + return err + } + alertHistoryRecord.Set("resolved", time.Now().UTC()) + err = app.Save(alertHistoryRecord) + if err != nil { + app.Logger().Error("Failed to resolve container alert history", "err", err) + } + return err +} + +// createContainerAlertHistoryRecord creates a new container alert history record +func createContainerAlertHistoryRecord(app core.App, alertRecord *core.Record) (alertHistoryRecord *core.Record, err error) { + alertHistoryCollection, err := app.FindCachedCollectionByNameOrId("alerts_history") + if err != nil { + return nil, err + } + alertHistoryRecord = core.NewRecord(alertHistoryCollection) + alertHistoryRecord.Set("alert_id", alertRecord.Id) + alertHistoryRecord.Set("user", alertRecord.GetString("user")) + alertHistoryRecord.Set("system", alertRecord.GetString("system")) + alertHistoryRecord.Set("container", alertRecord.GetString("container")) + alertHistoryRecord.Set("name", alertRecord.GetString("name")) + alertHistoryRecord.Set("value", alertRecord.GetFloat("value")) + err = app.Save(alertHistoryRecord) + if err != nil { + app.Logger().Error("Failed to save container alert history", "err", err) + } + return alertHistoryRecord, err +} diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 82b21f447..03fa6a334 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -296,6 +296,9 @@ func (h *Hub) registerApiRoutes(se *core.ServeEvent) error { // update / delete user alerts apiAuth.POST("/user-alerts", alerts.UpsertUserAlerts) apiAuth.DELETE("/user-alerts", alerts.DeleteUserAlerts) + // update / delete container alerts + apiAuth.POST("/user-container-alerts", alerts.UpsertUserContainerAlerts) + apiAuth.DELETE("/user-container-alerts", alerts.DeleteUserContainerAlerts) // refresh SMART devices for a system apiAuth.POST("/smart/refresh", h.refreshSmartData) // get systemd service details diff --git a/internal/hub/systems/system_manager.go b/internal/hub/systems/system_manager.go index 9dbe4b14f..09499b2dc 100644 --- a/internal/hub/systems/system_manager.go +++ b/internal/hub/systems/system_manager.go @@ -51,6 +51,7 @@ type hubLike interface { core.App GetSSHKey(dataDir string) (ssh.Signer, error) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error + HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData) error HandleStatusAlerts(status string, systemRecord *core.Record) error } @@ -211,6 +212,10 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error { if err := sm.hub.HandleSystemAlerts(e.Record, system.data); err != nil { e.App.Logger().Error("Error handling system alerts", "err", err) } + // Trigger container alerts + if err := sm.hub.HandleContainerAlerts(e.Record, system.data); err != nil { + e.App.Logger().Error("Error handling container alerts", "err", err) + } } // Trigger status change alerts for up/down transitions diff --git a/internal/migrations/1_container_alerts_0_19_0_dev.go b/internal/migrations/1_container_alerts_0_19_0_dev.go new file mode 100644 index 000000000..b2297045e --- /dev/null +++ b/internal/migrations/1_container_alerts_0_19_0_dev.go @@ -0,0 +1,208 @@ +package migrations + +import ( + "encoding/json" + + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + // 1. Create container_alerts collection + // Check if collection already exists to prevent errors on restart + if _, err := app.FindCollectionByNameOrId("container_alerts"); err == nil { + // Collection exists, skip creation + } else { + jsonData := `{ + "id": "ca_container_alerts", + "name": "container_alerts", + "type": "base", + "system": false, + "listRule": "@request.auth.id != \"\" && user.id = @request.auth.id", + "viewRule": "", + "createRule": "@request.auth.id != \"\" && user.id = @request.auth.id", + "updateRule": "@request.auth.id != \"\" && user.id = @request.auth.id", + "deleteRule": "@request.auth.id != \"\" && user.id = @request.auth.id", + "fields": [ + { + "id": "text_id", + "name": "id", + "type": "text", + "required": true, + "presentable": false, + "unique": false, + "system": true, + "primaryKey": true, + "autogeneratePattern": "[a-z0-9]{15}", + "min": 15, + "max": 15, + "pattern": "^[a-z0-9]+$" + }, + { + "id": "rel_user", + "name": "user", + "type": "relation", + "required": true, + "presentable": false, + "unique": false, + "system": false, + "collectionId": "_pb_users_auth_", + "cascadeDelete": true, + "minSelect": null, + "maxSelect": 1, + "displayFields": null + }, + { + "id": "rel_system", + "name": "system", + "type": "relation", + "required": true, + "presentable": false, + "unique": false, + "system": false, + "collectionId": "2hz5ncl8tizk5nx", + "cascadeDelete": true, + "minSelect": null, + "maxSelect": 1, + "displayFields": null + }, + { + "id": "text_container", + "name": "container", + "type": "text", + "required": true, + "presentable": false, + "unique": false, + "system": false, + "min": null, + "max": null, + "pattern": "" + }, + { + "id": "select_name", + "name": "name", + "type": "select", + "required": true, + "presentable": false, + "unique": false, + "system": false, + "maxSelect": 1, + "values": [ + "Status", + "CPU", + "Memory", + "Network", + "Health" + ] + }, + { + "id": "number_value", + "name": "value", + "type": "number", + "required": false, + "presentable": false, + "unique": false, + "system": false, + "min": null, + "max": null, + "onlyInt": false + }, + { + "id": "number_min", + "name": "min", + "type": "number", + "required": false, + "presentable": false, + "unique": false, + "system": false, + "min": null, + "max": 60, + "onlyInt": true + }, + { + "id": "bool_triggered", + "name": "triggered", + "type": "bool", + "required": false, + "presentable": false, + "unique": false, + "system": false + }, + { + "id": "autodate_created", + "name": "created", + "type": "autodate", + "required": false, + "presentable": false, + "unique": false, + "system": false, + "onCreate": true, + "onUpdate": false + }, + { + "id": "autodate_updated", + "name": "updated", + "type": "autodate", + "required": false, + "presentable": false, + "unique": false, + "system": false, + "onCreate": true, + "onUpdate": true + } + ], + "indexes": [ + "CREATE UNIQUE INDEX ` + "`" + `idx_container_alerts_unique` + "`" + ` ON ` + "`" + `container_alerts` + "`" + ` (` + "`" + `user` + "`" + `, ` + "`" + `system` + "`" + `, ` + "`" + `container` + "`" + `, ` + "`" + `name` + "`" + `)" + ] + }` + + collection := &core.Collection{} + if err := json.Unmarshal([]byte(jsonData), collection); err != nil { + return err + } + + if err := app.Save(collection); err != nil { + return err + } + } + + // 2. Add container field to alerts_history collection + alertsHistoryCollection, err := app.FindCollectionByNameOrId("alerts_history") + if err != nil { + return err + } + + if alertsHistoryCollection.Fields.GetByName("container") == nil { + containerField := &core.TextField{ + Name: "container", + Required: false, + } + + alertsHistoryCollection.Fields.Add(containerField) + + return app.Save(alertsHistoryCollection) + } + + return nil + }, func(app core.App) error { + // Rollback 2: remove the container field + alertsHistoryCollection, err := app.FindCollectionByNameOrId("alerts_history") + if err == nil { + if field := alertsHistoryCollection.Fields.GetByName("container"); field != nil { + alertsHistoryCollection.Fields.RemoveByName("container") + if err := app.Save(alertsHistoryCollection); err != nil { + return err + } + } + } + + // Rollback 1: delete the container_alerts collection + collection, err := app.FindCollectionByNameOrId("container_alerts") + if err == nil { + return app.Delete(collection) + } + + return nil + }) +} diff --git a/internal/site/src/components/alerts/container-alert-button.tsx b/internal/site/src/components/alerts/container-alert-button.tsx new file mode 100644 index 000000000..af5ceab0b --- /dev/null +++ b/internal/site/src/components/alerts/container-alert-button.tsx @@ -0,0 +1,44 @@ +import { t } from "@lingui/core/macro" +import { useStore } from "@nanostores/react" +import { BellIcon } from "lucide-react" +import { memo, useMemo, useState } from "react" +import { Button } from "@/components/ui/button" +import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet" +import { $containerAlerts } from "@/lib/stores" +import { cn } from "@/lib/utils" +import type { ContainerRecord } from "@/types" +import { ContainerAlertDialogContent } from "./container-alerts-sheet" + +export default memo(function ContainerAlertButton({ + systemId, + container, +}: { + systemId: string + container: ContainerRecord +}) { + const [opened, setOpened] = useState(false) + const alerts = useStore($containerAlerts) + + const containerAlerts = alerts[systemId]?.get(container.id) + const hasContainerAlert = containerAlerts && containerAlerts.size > 0 + + return useMemo( + () => ( + + + + + + {opened && } + + + ), + [opened, hasContainerAlert] + ) +}) diff --git a/internal/site/src/components/alerts/container-alerts-sheet.tsx b/internal/site/src/components/alerts/container-alerts-sheet.tsx new file mode 100644 index 000000000..5bd1d972b --- /dev/null +++ b/internal/site/src/components/alerts/container-alerts-sheet.tsx @@ -0,0 +1,327 @@ +import { t } from "@lingui/core/macro" +import { Plural, Trans } from "@lingui/react/macro" +import { useStore } from "@nanostores/react" +import { getPagePath } from "@nanostores/router" +import { BoxIcon, GlobeIcon } from "lucide-react" +import { lazy, memo, Suspense, useMemo, useState } from "react" +import { $router, Link } from "@/components/router" +import { Checkbox } from "@/components/ui/checkbox" +import { DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog" +import { Switch } from "@/components/ui/switch" +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" +import { toast } from "@/components/ui/use-toast" +import { containerAlertInfo } from "@/lib/container-alerts" +import { pb } from "@/lib/api" +import { $containerAlerts } from "@/lib/stores" +import { cn, debounce } from "@/lib/utils" +import type { ContainerAlertInfo, ContainerAlertRecord, ContainerRecord } from "@/types" + +const Slider = lazy(() => import("@/components/ui/slider")) + +const endpoint = "/api/beszel/user-container-alerts" + +const alertDebounce = 100 + +const alertKeys = Object.keys(containerAlertInfo) as (keyof typeof containerAlertInfo)[] + +const failedUpdateToast = (error: unknown) => { + console.error(error) + toast({ + title: t`Failed to update alert`, + description: t`Please check logs for more details.`, + variant: "destructive", + }) +} + +/** Create or update container alerts */ +const upsertContainerAlerts = debounce( + async ({ + name, + value, + min, + systems, + containers, + }: { + name: string + value: number + min: number + systems: string[] + containers: string[] + }) => { + try { + await pb.send<{ success: boolean }>(endpoint, { + method: "POST", + body: { name, value, min, systems, containers, overwrite: true }, + }) + } catch (error) { + failedUpdateToast(error) + } + }, + alertDebounce +) + +/** Delete container alerts */ +const deleteContainerAlerts = debounce( + async ({ name, systems, containers }: { name: string; systems: string[]; containers: string[] }) => { + try { + await pb.send<{ success: boolean }>(endpoint, { + method: "DELETE", + body: { name, systems, containers }, + }) + } catch (error) { + failedUpdateToast(error) + } + }, + alertDebounce +) + +export const ContainerAlertDialogContent = memo(function ContainerAlertDialogContent({ + systemId, + container, +}: { + systemId: string + container: ContainerRecord +}) { + const alerts = useStore($containerAlerts) + const [overwriteExisting, setOverwriteExisting] = useState(false) + const [currentTab, setCurrentTab] = useState("container") + + const containerAlerts = alerts[systemId]?.get(container.id) ?? new Map() + + // Keep a copy of alerts when we switch to global tab + const alertsWhenGlobalSelected = useMemo(() => { + return currentTab === "global" ? structuredClone(alerts) : alerts + }, [currentTab]) + + return ( + <> + + + Container Alerts + + + + See{" "} + + notification settings + {" "} + to configure how you receive alerts. + + + + + + + + {container.name} + + + + All Containers + + + +
+ {alertKeys.map((name) => ( + + ))} +
+
+ + +
+ {alertKeys.map((name) => ( + + ))} +
+
+
+ + ) +}) + +export function ContainerAlertContent({ + alertKey, + data: alertData, + systemId, + container, + alert, + global = false, + overwriteExisting = false, + initialAlertsState = {}, +}: { + alertKey: string + data: ContainerAlertInfo + systemId: string + container: ContainerRecord + alert?: ContainerAlertRecord + global?: boolean + overwriteExisting?: boolean + initialAlertsState?: Record>> +}) { + const { name } = alertData + + const singleDescription = alertData.singleDesc?.() + + const [checked, setChecked] = useState(global ? false : !!alert) + const [min, setMin] = useState(alert?.min || 10) + const [value, setValue] = useState(alert?.value || (singleDescription ? 0 : alertData.start ?? 80)) + + const Icon = alertData.icon + + /** Get container ids to update */ + function getContainerIds(): string[] { + // if not global, update only the current container + if (!global) { + return [container.id] + } + // if global, we need to get all containers for this system + // For now, we'll just use the current container + // In a real implementation, you'd fetch all containers for the system + return [container.id] + } + + function sendUpsert(min: number, value: number) { + const containers = getContainerIds() + containers.length && + upsertContainerAlerts({ + name: alertKey, + value, + min, + systems: [systemId], + containers, + }) + } + + return ( +
+ + {checked && ( +
+ }> + {!singleDescription && ( +
+

+ {alertData.invert ? ( + + Average drops below{" "} + + {value} + {alertData.unit} + + + ) : ( + + Average exceeds{" "} + + {value} + {alertData.unit} + + + )} +

+
+ sendUpsert(min, val[0])} + onValueChange={(val) => setValue(val[0])} + step={alertData.step ?? 1} + min={alertData.min ?? 1} + max={alertData.max ?? 99} + /> +
+
+ )} +
+

+ {singleDescription && ( + <> + {singleDescription} + {` `} + + )} + + For {min}{" "} + + +

+
+ sendUpsert(minVal[0], value)} + onValueChange={(val) => setMin(val[0])} + min={1} + max={60} + /> +
+
+
+
+ )} +
+ ) +} diff --git a/internal/site/src/components/containers-table/containers-table-columns.tsx b/internal/site/src/components/containers-table/containers-table-columns.tsx index 80d70ccbe..dda9c8b53 100644 --- a/internal/site/src/components/containers-table/containers-table-columns.tsx +++ b/internal/site/src/components/containers-table/containers-table-columns.tsx @@ -1,4 +1,5 @@ import type { Column, ColumnDef } from "@tanstack/react-table" +import { lazy, Suspense } from "react" import { Button } from "@/components/ui/button" import { cn, decimalString, formatBytes, hourWithSeconds } from "@/lib/utils" import type { ContainerRecord } from "@/types" @@ -40,6 +41,9 @@ function getStatusValue(status: string): number { return 0 } +// Lazy load the alert button component +const ContainerAlertButton = lazy(() => import("@/components/alerts/container-alert-button")) + export const containerChartCols: ColumnDef[] = [ { id: "name", @@ -62,7 +66,7 @@ export const containerChartCols: ColumnDef[] = [ header: ({ column }) => , cell: ({ getValue }) => { const allSystems = useStore($allSystemsById) - return {allSystems[getValue() as string]?.name ?? ""} + return {allSystems[getValue() as string]?.name ?? ""} }, }, // { @@ -137,17 +141,17 @@ export const containerChartCols: ColumnDef[] = [ id: "image", sortingFn: (a, b) => a.original.image.localeCompare(b.original.image), accessorFn: (record) => record.image, - header: ({ column }) => ( - - ), + enableHiding: true, + header: ({ column }) => , cell: ({ getValue }) => { - return {getValue() as string} + return {getValue() as string} }, }, { id: "status", accessorFn: (record) => record.status, invertSorting: true, + enableHiding: true, sortingFn: (a, b) => getStatusValue(a.original.status) - getStatusValue(b.original.status), header: ({ column }) => , cell: ({ getValue }) => { @@ -164,6 +168,21 @@ export const containerChartCols: ColumnDef[] = [ return {hourWithSeconds(new Date(timestamp).toISOString())} }, }, + { + id: "actions", + header: () =>
{t`Actions`}
, + cell: ({ row }) => { + // Lazy load the alert button component + const ContainerAlertButton = lazy(() => import("@/components/alerts/container-alert-button")) + return ( +
e.stopPropagation()}> + }> + + +
+ ) + }, + }, ] function HeaderButton({ diff --git a/internal/site/src/components/containers-table/containers-table.tsx b/internal/site/src/components/containers-table/containers-table.tsx index 92c63bebf..952201261 100644 --- a/internal/site/src/components/containers-table/containers-table.tsx +++ b/internal/site/src/components/containers-table/containers-table.tsx @@ -231,7 +231,7 @@ const AllContainersTable = memo(function AllContainersTable({ > {/* add header height to table size */}
- +
{rows.length ? ( diff --git a/internal/site/src/lib/api.ts b/internal/site/src/lib/api.ts index 79f4b1279..5abf631af 100644 --- a/internal/site/src/lib/api.ts +++ b/internal/site/src/lib/api.ts @@ -3,7 +3,7 @@ import PocketBase from "pocketbase" import { basePath } from "@/components/router" import { toast } from "@/components/ui/use-toast" import type { ChartTimes, UserSettings } from "@/types" -import { $alerts, $allSystemsById, $allSystemsByName, $userSettings } from "./stores" +import { $alerts, $allSystemsById, $allSystemsByName, $containerAlerts, $userSettings } from "./stores" import { chartTimeData } from "./utils" /** PocketBase JS Client */ @@ -30,6 +30,7 @@ export function logOut() { $allSystemsByName.set({}) $allSystemsById.set({}) $alerts.set({}) + $containerAlerts.set({}) $userSettings.set({} as UserSettings) sessionStorage.setItem("lo", "t") // prevent auto login on logout pb.authStore.clear() diff --git a/internal/site/src/lib/container-alerts.ts b/internal/site/src/lib/container-alerts.ts new file mode 100644 index 000000000..2cdc2d48f --- /dev/null +++ b/internal/site/src/lib/container-alerts.ts @@ -0,0 +1,180 @@ +import { t } from "@lingui/core/macro" +import { CpuIcon, HeartPulseIcon, MemoryStickIcon, ServerIcon } from "lucide-react" +import type { RecordSubscription } from "pocketbase" +import { EthernetIcon } from "@/components/ui/icons" +import { $containerAlerts } from "@/lib/stores" +import type { ContainerAlertInfo, ContainerAlertRecord } from "@/types" +import { pb } from "./api" + +/** Container alert info for each alert type */ +export const containerAlertInfo: Record = { + Status: { + name: () => t`Status`, + unit: "", + icon: ServerIcon, + desc: () => t`Triggers when container status changes`, + singleDesc: () => `${t`Container`} ${t`Stopped`}`, + }, + CPU: { + name: () => t`CPU Usage`, + unit: "%", + icon: CpuIcon, + desc: () => t`Triggers when CPU usage exceeds a threshold`, + }, + Memory: { + name: () => t`Memory Usage`, + unit: "%", + icon: MemoryStickIcon, + desc: () => t`Triggers when memory usage exceeds a threshold`, + }, + Network: { + name: () => t`Network`, + unit: " MB/s", + icon: EthernetIcon, + desc: () => t`Triggers when combined up/down exceeds a threshold`, + max: 125, + }, + Health: { + name: () => t`Health Status`, + unit: "", + icon: HeartPulseIcon, + desc: () => t`Triggers when container health status changes`, + singleDesc: () => `${t`Container`} ${t`Unhealthy`}`, + }, +} + +class ContainerAlertManager { + private unsubscribeFn?: () => void + + /** + * Subscribe to container alert updates + */ + async subscribe() { + if (this.unsubscribeFn) { + return + } + + // Fetch initial container alerts + try { + const alerts = await pb.collection("container_alerts").getFullList() + this.updateStore(alerts) + } catch (e) { + console.error("Failed to fetch container alerts:", e) + } + + // Subscribe to real-time updates + this.unsubscribeFn = await pb + .collection("container_alerts") + .subscribe("*", this.handleAlertUpdate) + } + + /** + * Unsubscribe from container alert updates + */ + unsubscribe() { + if (this.unsubscribeFn) { + this.unsubscribeFn() + this.unsubscribeFn = undefined + } + } + + /** + * Handle real-time alert updates + */ + private handleAlertUpdate = (e: RecordSubscription) => { + const { action, record } = e + + if (action === "delete") { + this.deleteFromStore(record) + } else { + this.updateStore([record]) + } + } + + /** + * Update store with alert records + */ + private updateStore(alerts: ContainerAlertRecord[]) { + const currentAlerts = $containerAlerts.get() + + for (const alert of alerts) { + if (!currentAlerts[alert.system]) { + currentAlerts[alert.system] = new Map() + } + if (!currentAlerts[alert.system].get(alert.container)) { + currentAlerts[alert.system].set(alert.container, new Map()) + } + currentAlerts[alert.system].get(alert.container)!.set(alert.name, alert) + } + + $containerAlerts.set(currentAlerts) + } + + /** + * Delete alert from store + */ + private deleteFromStore(alert: ContainerAlertRecord) { + const currentAlerts = $containerAlerts.get() + + if (currentAlerts[alert.system]?.get(alert.container)?.has(alert.name)) { + currentAlerts[alert.system].get(alert.container)!.delete(alert.name) + + // Clean up empty maps + if (currentAlerts[alert.system].get(alert.container)!.size === 0) { + currentAlerts[alert.system].delete(alert.container) + } + if (currentAlerts[alert.system].size === 0) { + delete currentAlerts[alert.system] + } + + $containerAlerts.set(currentAlerts) + } + } + + /** + * Create or update container alerts + */ + async upsert( + systems: string[], + containers: string[], + name: string, + value: number, + min: number, + overwrite = false + ) { + return pb.send("/api/beszel/user-container-alerts", { + method: "POST", + body: { + systems, + containers, + name, + value, + min, + overwrite, + }, + }) + } + + /** + * Delete container alerts + */ + async delete(systems: string[], containers: string[], name: string) { + return pb.send("/api/beszel/user-container-alerts", { + method: "DELETE", + body: { + systems, + containers, + name, + }, + }) + } + + /** + * Clear all container alerts from store + */ + clear() { + $containerAlerts.set({}) + } +} + +export const containerAlertManager = new ContainerAlertManager() diff --git a/internal/site/src/lib/stores.ts b/internal/site/src/lib/stores.ts index 9cc564b48..4ca670f43 100644 --- a/internal/site/src/lib/stores.ts +++ b/internal/site/src/lib/stores.ts @@ -1,5 +1,5 @@ import { atom, computed, listenKeys, map, type ReadableAtom } from "nanostores" -import type { AlertMap, ChartTimes, SystemRecord, UserSettings } from "@/types" +import type { AlertMap, ChartTimes, ContainerAlertMap, SystemRecord, UserSettings } from "@/types" import { pb } from "./api" import { Unit } from "./enums" @@ -22,6 +22,9 @@ export const $systems: ReadableAtom = computed($allSystemsById, /** Map of alert records by system id and alert name */ export const $alerts = map({}) +/** Map of container alert records by system id, container id, and alert name */ +export const $containerAlerts = map({}) + /** SSH public key */ export const $publicKey = atom("") diff --git a/internal/site/src/main.tsx b/internal/site/src/main.tsx index 81cd3c63c..377d6fe50 100644 --- a/internal/site/src/main.tsx +++ b/internal/site/src/main.tsx @@ -12,6 +12,7 @@ import Settings from "@/components/routes/settings/layout.tsx" import { ThemeProvider } from "@/components/theme-provider.tsx" import { Toaster } from "@/components/ui/toaster.tsx" import { alertManager } from "@/lib/alerts" +import { containerAlertManager } from "@/lib/container-alerts" import { pb, updateUserSettings } from "@/lib/api.ts" import { dynamicActivate, getLocale } from "@/lib/i18n" import { $authenticated, $copyContent, $direction, $publicKey, $userSettings } from "@/lib/stores.ts" @@ -49,8 +50,11 @@ const App = memo(() => { .then(alertManager.refresh) // subscribe to new alert updates .then(alertManager.subscribe) + // subscribe to container alerts + .then(() => containerAlertManager.subscribe()) return () => { alertManager.unsubscribe() + containerAlertManager.unsubscribe() systemsManager.unsubscribe() } }, []) diff --git a/internal/site/src/types.d.ts b/internal/site/src/types.d.ts index 53dce80c0..b14c7452d 100644 --- a/internal/site/src/types.d.ts +++ b/internal/site/src/types.d.ts @@ -340,6 +340,33 @@ export interface AlertInfo { export type AlertMap = Record> +export interface ContainerAlertRecord extends RecordModel { + id: string + system: string + container: string + name: string + triggered: boolean + value: number + min: number +} + +export interface ContainerAlertInfo { + name: () => string + unit: string + icon: any + desc: () => string + max?: number + min?: number + step?: number + start?: number + /** Single value description (when there's only one value, like status) */ + singleDesc?: () => string + invert?: boolean +} + +export type ContainerAlertMap = Record>> + + export interface SmartData { /** model family */ // mf?: string