mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-18 22:14:28 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
065d7f7f01 |
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
# Node.js dependencies
|
# Node.js dependencies
|
||||||
node_modules/
|
node_modules
|
||||||
**/node_modules/
|
internalsite/node_modules
|
||||||
|
|
||||||
# Go build artifacts and binaries
|
# Go build artifacts and binaries
|
||||||
build
|
build
|
||||||
|
|||||||
@@ -193,7 +193,7 @@ jobs:
|
|||||||
uses: docker/setup-qemu-action@v4
|
uses: docker/setup-qemu-action@v4
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
- name: Docker metadata
|
- name: Docker metadata
|
||||||
id: metadata
|
id: metadata
|
||||||
|
|||||||
@@ -1,109 +0,0 @@
|
|||||||
name: Helm charts
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- "supplemental/helm/**"
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- "supplemental/helm/**"
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
packages: write
|
|
||||||
|
|
||||||
env:
|
|
||||||
OCI_REGISTRY: ghcr.io/henrygd/beszel-charts
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
changes:
|
|
||||||
name: Detect changed charts
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
outputs:
|
|
||||||
charts: ${{ steps.changes.outputs.charts }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v7
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- name: Detect changed charts
|
|
||||||
id: changes
|
|
||||||
env:
|
|
||||||
BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
|
|
||||||
run: |
|
|
||||||
charts=()
|
|
||||||
|
|
||||||
for name in beszel-agent beszel-hub; do
|
|
||||||
path="supplemental/helm/$name"
|
|
||||||
if ! git diff --quiet "$BASE_SHA" "$GITHUB_SHA" -- "$path"; then
|
|
||||||
charts+=("$name|$path")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
printf '%s\n' "${charts[@]}" \
|
|
||||||
| jq -Rsc 'split("\n") | map(select(length > 0) | split("|") | {name: .[0], path: .[1]})' \
|
|
||||||
| xargs -0 printf 'charts=%s\n' >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
validate-and-publish:
|
|
||||||
name: ${{ github.event_name == 'push' && 'Publish' || 'Validate' }} ${{ matrix.chart.name }}
|
|
||||||
needs: changes
|
|
||||||
if: needs.changes.outputs.charts != '[]'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
chart: ${{ fromJSON(needs.changes.outputs.charts) }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v7
|
|
||||||
|
|
||||||
- name: Set up Helm
|
|
||||||
uses: azure/setup-helm@v5
|
|
||||||
|
|
||||||
- name: Lint chart
|
|
||||||
run: helm lint "${{ matrix.chart.path }}" --set env.KEY=ci-placeholder
|
|
||||||
|
|
||||||
- name: Render chart
|
|
||||||
run: helm template "${{ matrix.chart.name }}" "${{ matrix.chart.path }}" --set env.KEY=ci-placeholder > /dev/null
|
|
||||||
|
|
||||||
- name: Package chart
|
|
||||||
id: package
|
|
||||||
env:
|
|
||||||
CHART_NAME: ${{ matrix.chart.name }}
|
|
||||||
CHART_PATH: ${{ matrix.chart.path }}
|
|
||||||
run: |
|
|
||||||
version=$(awk '/^version:/ { print $2 }' "$CHART_PATH/Chart.yaml")
|
|
||||||
test -n "$version"
|
|
||||||
|
|
||||||
mkdir -p .helm-packages
|
|
||||||
helm package "$CHART_PATH" --destination .helm-packages
|
|
||||||
|
|
||||||
package=".helm-packages/${CHART_NAME}-${version}.tgz"
|
|
||||||
test -f "$package"
|
|
||||||
echo "version=$version" >> "$GITHUB_OUTPUT"
|
|
||||||
echo "package=$package" >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
- name: Log in to GHCR
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ github.token }}
|
|
||||||
run: echo "$GITHUB_TOKEN" | helm registry login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
|
|
||||||
|
|
||||||
- name: Check chart version is unpublished
|
|
||||||
env:
|
|
||||||
CHART_NAME: ${{ matrix.chart.name }}
|
|
||||||
CHART_VERSION: ${{ steps.package.outputs.version }}
|
|
||||||
run: |
|
|
||||||
chart="oci://${OCI_REGISTRY}/${CHART_NAME}"
|
|
||||||
if helm show chart "$chart" --version "$CHART_VERSION" > /dev/null 2>&1; then
|
|
||||||
echo "${CHART_NAME} ${CHART_VERSION} is already published. Bump version in Chart.yaml." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Publish chart
|
|
||||||
if: github.event_name == 'push'
|
|
||||||
run: helm push "${{ steps.package.outputs.package }}" "oci://${OCI_REGISTRY}"
|
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- ".github/workflows/release-please.yml"
|
||||||
|
- "release-please-config.json"
|
||||||
|
- ".release-please-manifest.json"
|
||||||
|
- "supplemental/helm/**"
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
issues: write
|
||||||
|
pull-requests: write
|
||||||
|
|
||||||
|
env:
|
||||||
|
PUBLISHABLE_ITEMS: '["supplemental/helm/beszel-agent", "supplemental/helm/beszel-hub"]'
|
||||||
|
|
||||||
|
name: helm-release
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: helm-release
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release:
|
||||||
|
name: Release
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- uses: googleapis/release-please-action@v4
|
||||||
|
name: Prepare release
|
||||||
|
id: release-please
|
||||||
|
with:
|
||||||
|
token: ${{ secrets.CR_TOKEN }}
|
||||||
|
|
||||||
|
- name: Dump Release Please Output
|
||||||
|
env:
|
||||||
|
RELEASE_PLEASE_OUTPUT: ${{ toJson(steps.release-please.outputs) }}
|
||||||
|
run: |
|
||||||
|
echo "$RELEASE_PLEASE_OUTPUT"
|
||||||
|
|
||||||
|
- name: Determine what should be published
|
||||||
|
uses: actions/github-script@v8
|
||||||
|
id: items-to-publish
|
||||||
|
env:
|
||||||
|
CHANGED_ITEMS: "${{ steps.release-please.outputs.paths_released }}"
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const changedItems = JSON.parse(process.env.CHANGED_ITEMS || '[]');
|
||||||
|
console.log("changed items", changedItems);
|
||||||
|
const eligibleItems = JSON.parse(process.env.PUBLISHABLE_ITEMS || '[]');
|
||||||
|
console.log("eligible items", eligibleItems);
|
||||||
|
const itemsToPublish = changedItems.filter(i => eligibleItems.includes(i));
|
||||||
|
console.log("items to publish", itemsToPublish);
|
||||||
|
return itemsToPublish;
|
||||||
|
outputs:
|
||||||
|
items_to_publish: ${{ steps.items-to-publish.outputs.result }}
|
||||||
|
releases: ${{ toJson(steps.release-please.outputs) }}
|
||||||
|
|
||||||
|
release-charts:
|
||||||
|
name: Release Charts
|
||||||
|
needs: release
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: ${{ needs.release.outputs.items_to_publish != '' && toJson(fromJson(needs.release.outputs.items_to_publish)) != '[]' }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
max-parallel: 1
|
||||||
|
matrix:
|
||||||
|
path: ${{ fromJSON(needs.release.outputs.items_to_publish) }}
|
||||||
|
|
||||||
|
env:
|
||||||
|
TAG: ${{ fromJson(needs.release.outputs.releases)[format('{0}--tag_name', matrix.path)] }}
|
||||||
|
VERSION: ${{ fromJson(needs.release.outputs.releases)[format('{0}--version', matrix.path)] }}
|
||||||
|
RELEASE_ID: ${{ fromJson(needs.release.outputs.releases)[format('{0}--id', matrix.path)] }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Debug
|
||||||
|
run: |
|
||||||
|
echo ${{ env.TAG }}
|
||||||
|
echo ${{ env.VERSION }}
|
||||||
|
echo ${{ matrix.path }}
|
||||||
|
|
||||||
|
- name: ✨ Checkout repository
|
||||||
|
uses: actions/checkout@v5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Configure Git
|
||||||
|
run: |
|
||||||
|
echo ${{ needs.release.outputs.items_to_publish }}
|
||||||
|
git config user.name "$GITHUB_ACTOR"
|
||||||
|
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
|
||||||
|
|
||||||
|
- name: Install Chart Releaser
|
||||||
|
run: |
|
||||||
|
version=v1.6.0
|
||||||
|
mkdir .tmp
|
||||||
|
install_dir=.tmp
|
||||||
|
echo "Installing chart-releaser on $install_dir..."
|
||||||
|
curl -sSLo cr.tar.gz "https://github.com/helm/chart-releaser/releases/download/$version/chart-releaser_${version#v}_linux_amd64.tar.gz"
|
||||||
|
tar -xzf cr.tar.gz -C "$install_dir"
|
||||||
|
rm -f cr.tar.gz
|
||||||
|
|
||||||
|
- name: Package chart
|
||||||
|
run: |
|
||||||
|
.tmp/cr package ${{ matrix.path }}
|
||||||
|
ls -la .cr-release-packages/
|
||||||
|
|
||||||
|
- name: Upload chart package to draft release
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
run: gh release upload "$TAG" .cr-release-packages/*.tgz --clobber
|
||||||
|
|
||||||
|
- name: Publish chart release
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
CHART_PATH: ${{ matrix.path }}
|
||||||
|
run: |
|
||||||
|
chart_name="${CHART_PATH##*/}"
|
||||||
|
gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \
|
||||||
|
-f "name=Helm chart: ${chart_name} v${VERSION}" \
|
||||||
|
-F draft=false \
|
||||||
|
-f make_latest=false
|
||||||
|
|
||||||
|
- name: Update index
|
||||||
|
run: |
|
||||||
|
owner=$(cut -d '/' -f 1 <<< "$GITHUB_REPOSITORY")
|
||||||
|
repo=$(cut -d '/' -f 2 <<< "$GITHUB_REPOSITORY")
|
||||||
|
|
||||||
|
# Create the local output directory for the generated index
|
||||||
|
mkdir -p docs
|
||||||
|
|
||||||
|
# Generate index.yaml for GitHub Pages
|
||||||
|
args=(-o "$owner" -r "$repo" --push -t "${{ secrets.CR_TOKEN }}" --index-path docs/index.yaml --release-name-template '{{ .Name }}-v{{ .Version }}')
|
||||||
|
.tmp/cr index "${args[@]}"
|
||||||
|
|
||||||
|
- name: Restore application release as latest
|
||||||
|
if: ${{ always() }}
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
run: |
|
||||||
|
release_id=$(gh api "repos/${GITHUB_REPOSITORY}/releases?per_page=100" \
|
||||||
|
--jq 'map(select(.draft == false and .prerelease == false and (.tag_name | test("^v[0-9]"))))[0].id')
|
||||||
|
test -n "$release_id"
|
||||||
|
gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${release_id}" \
|
||||||
|
-f make_latest=true
|
||||||
@@ -29,7 +29,7 @@ jobs:
|
|||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v7
|
uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: stable
|
go-version: "^1.22.1"
|
||||||
|
|
||||||
- name: Set up .NET
|
- name: Set up .NET
|
||||||
uses: actions/setup-dotnet@v6
|
uses: actions/setup-dotnet@v6
|
||||||
@@ -52,3 +52,51 @@ jobs:
|
|||||||
GITHUB_TOKEN: ${{ secrets.TOKEN || secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.TOKEN || secrets.GITHUB_TOKEN }}
|
||||||
WINGET_TOKEN: ${{ secrets.WINGET_TOKEN }}
|
WINGET_TOKEN: ${{ secrets.WINGET_TOKEN }}
|
||||||
IS_FORK: ${{ github.repository_owner != 'henrygd' }}
|
IS_FORK: ${{ github.repository_owner != 'henrygd' }}
|
||||||
|
|
||||||
|
update-helm-app-version:
|
||||||
|
name: Update Helm app version
|
||||||
|
needs: goreleaser
|
||||||
|
if: ${{ github.repository_owner == 'henrygd' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout main
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
ref: main
|
||||||
|
token: ${{ secrets.CR_TOKEN }}
|
||||||
|
|
||||||
|
- name: Update chart app versions
|
||||||
|
env:
|
||||||
|
RELEASE_TAG: ${{ github.ref_name }}
|
||||||
|
run: |
|
||||||
|
version="${RELEASE_TAG#v}"
|
||||||
|
if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$ ]]; then
|
||||||
|
echo "Invalid release version: $version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for chart in supplemental/helm/beszel-agent supplemental/helm/beszel-hub; do
|
||||||
|
current_version=$(awk -F '"' '/^appVersion:/ { print $2 }' "$chart/Chart.yaml")
|
||||||
|
NEW_VERSION="$version" perl -pi -e 's/^appVersion:.*$/appVersion: "$ENV{NEW_VERSION}"/' "$chart/Chart.yaml"
|
||||||
|
OLD_VERSION="$current_version" NEW_VERSION="$version" \
|
||||||
|
perl -pi -e 's/\Q$ENV{OLD_VERSION}\E/$ENV{NEW_VERSION}/g' "$chart/README.md"
|
||||||
|
done
|
||||||
|
|
||||||
|
- name: Commit app version update
|
||||||
|
env:
|
||||||
|
RELEASE_TAG: ${{ github.ref_name }}
|
||||||
|
run: |
|
||||||
|
if git diff --quiet -- supplemental/helm; then
|
||||||
|
echo "Helm charts already use ${RELEASE_TAG#v}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
git config user.name "github-actions[bot]"
|
||||||
|
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||||
|
git add supplemental/helm/beszel-agent/Chart.yaml \
|
||||||
|
supplemental/helm/beszel-agent/README.md \
|
||||||
|
supplemental/helm/beszel-hub/Chart.yaml \
|
||||||
|
supplemental/helm/beszel-hub/README.md
|
||||||
|
git commit -m "fix(helm): update app version to ${RELEASE_TAG#v}"
|
||||||
|
git push origin HEAD:main
|
||||||
|
|||||||
@@ -1,101 +0,0 @@
|
|||||||
name: Update Helm charts
|
|
||||||
|
|
||||||
on:
|
|
||||||
release:
|
|
||||||
types:
|
|
||||||
- published
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
pull-requests: write
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: update-helm-charts
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
update:
|
|
||||||
name: Propose chart update
|
|
||||||
if: ${{ github.repository_owner == 'henrygd' && startsWith(github.event.release.tag_name, 'v') && !github.event.release.prerelease }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
env:
|
|
||||||
BRANCH: automation/update-helm-app-version
|
|
||||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
||||||
AUTOMATION_TOKEN: ${{ secrets.CR_TOKEN || github.token }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout main
|
|
||||||
uses: actions/checkout@v7
|
|
||||||
with:
|
|
||||||
ref: main
|
|
||||||
token: ${{ env.AUTOMATION_TOKEN }}
|
|
||||||
|
|
||||||
- name: Update chart versions
|
|
||||||
id: update
|
|
||||||
run: |
|
|
||||||
version="${RELEASE_TAG#v}"
|
|
||||||
if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
|
||||||
echo "Unsupported software release version: $version" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
changed=false
|
|
||||||
for chart in supplemental/helm/beszel-agent supplemental/helm/beszel-hub; do
|
|
||||||
current_app_version=$(awk -F '"' '/^appVersion:/ { print $2 }' "$chart/Chart.yaml")
|
|
||||||
if [[ "$current_app_version" == "$version" ]]; then
|
|
||||||
echo "$chart already uses appVersion $version"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
newest_version=$(printf '%s\n' "$current_app_version" "$version" | sort -V | tail -n 1)
|
|
||||||
if [[ "$newest_version" != "$version" ]]; then
|
|
||||||
echo "Skipping stale update of $chart from $current_app_version to $version"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
chart_version=$(awk '/^version:/ { print $2 }' "$chart/Chart.yaml")
|
|
||||||
if [[ ! "$chart_version" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
|
||||||
echo "Unsupported chart version in $chart/Chart.yaml: $chart_version" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
next_chart_version="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}.$((BASH_REMATCH[3] + 1))"
|
|
||||||
|
|
||||||
NEW_APP_VERSION="$version" NEW_CHART_VERSION="$next_chart_version" \
|
|
||||||
perl -pi -e 's/^appVersion:.*$/appVersion: "$ENV{NEW_APP_VERSION}"/; s/^version:.*$/version: $ENV{NEW_CHART_VERSION}/' \
|
|
||||||
"$chart/Chart.yaml"
|
|
||||||
OLD_APP_VERSION="$current_app_version" NEW_APP_VERSION="$version" \
|
|
||||||
perl -pi -e 's/\Q$ENV{OLD_APP_VERSION}\E/$ENV{NEW_APP_VERSION}/g' "$chart/README.md"
|
|
||||||
|
|
||||||
echo "$chart: appVersion $current_app_version -> $version, chart $chart_version -> $next_chart_version"
|
|
||||||
changed=true
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "changed=$changed" >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
- name: Open or update pull request
|
|
||||||
if: steps.update.outputs.changed == 'true'
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ env.AUTOMATION_TOKEN }}
|
|
||||||
run: |
|
|
||||||
version="${RELEASE_TAG#v}"
|
|
||||||
title="chore(helm): update app version to ${version}"
|
|
||||||
body="Updates the Helm charts for [Beszel ${version}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/tag/${RELEASE_TAG}) and bumps their chart patch versions. Merging this pull request publishes the updated charts to GHCR."
|
|
||||||
|
|
||||||
git config user.name "github-actions[bot]"
|
|
||||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
||||||
git checkout -B "$BRANCH"
|
|
||||||
git add supplemental/helm/beszel-agent/Chart.yaml \
|
|
||||||
supplemental/helm/beszel-agent/README.md \
|
|
||||||
supplemental/helm/beszel-hub/Chart.yaml \
|
|
||||||
supplemental/helm/beszel-hub/README.md
|
|
||||||
git commit -m "$title"
|
|
||||||
|
|
||||||
git fetch origin "$BRANCH" || true
|
|
||||||
git push --force-with-lease origin "HEAD:refs/heads/${BRANCH}"
|
|
||||||
|
|
||||||
pr_number=$(gh pr list --head "$BRANCH" --base main --state open --json number --jq '.[0].number')
|
|
||||||
if [[ -n "$pr_number" ]]; then
|
|
||||||
gh pr edit "$pr_number" --title "$title" --body "$body"
|
|
||||||
else
|
|
||||||
gh pr create --base main --head "$BRANCH" --title "$title" --body "$body"
|
|
||||||
fi
|
|
||||||
@@ -2,6 +2,10 @@
|
|||||||
|
|
||||||
name: VulnCheck
|
name: VulnCheck
|
||||||
on:
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
|||||||
+3
-3
@@ -22,6 +22,9 @@ builds:
|
|||||||
- amd64
|
- amd64
|
||||||
- arm64
|
- arm64
|
||||||
- arm
|
- arm
|
||||||
|
goarm:
|
||||||
|
- "6"
|
||||||
|
- "7"
|
||||||
ignore:
|
ignore:
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: arm64
|
goarch: arm64
|
||||||
@@ -39,8 +42,6 @@ builds:
|
|||||||
main: internal/cmd/agent/agent.go
|
main: internal/cmd/agent/agent.go
|
||||||
env:
|
env:
|
||||||
- CGO_ENABLED=0
|
- CGO_ENABLED=0
|
||||||
ldflags:
|
|
||||||
- -s -w -X github.com/henrygd/beszel/internal/ghupdate.buildGOARM={{ .Arm }}
|
|
||||||
goos:
|
goos:
|
||||||
- linux
|
- linux
|
||||||
- darwin
|
- darwin
|
||||||
@@ -107,7 +108,6 @@ archives:
|
|||||||
{{ .Binary }}_
|
{{ .Binary }}_
|
||||||
{{- .Os }}_
|
{{- .Os }}_
|
||||||
{{- .Arch }}
|
{{- .Arch }}
|
||||||
{{- if ne .Arm "6" }}{{ with .Arm }}v{{ . }}{{ end }}{{ end }}
|
|
||||||
format_overrides:
|
format_overrides:
|
||||||
- goos: windows
|
- goos: windows
|
||||||
formats: [zip]
|
formats: [zip]
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"supplemental/helm/beszel-agent": "0.1.4",
|
||||||
|
"supplemental/helm/beszel-hub": "0.1.4"
|
||||||
|
}
|
||||||
+2
-4
@@ -2,8 +2,6 @@
|
|||||||
|
|
||||||
## Reporting a Vulnerability
|
## Reporting a Vulnerability
|
||||||
|
|
||||||
**PLEASE ONLY USE SECURITY ADVISORIES FOR REAL HIGH SEVERITY VULNERABILITIES.**
|
If you find a vulnerability in the latest version, please [submit a private advisory](https://github.com/henrygd/beszel/security/advisories/new).
|
||||||
|
|
||||||
If you find a vulnerability in the latest version, and it is not high severity, open an issue instead of an advisory.
|
If it's low severity (use best judgement) you may open an issue instead of an advisory.
|
||||||
|
|
||||||
I am overwhelmed with advisories, often erroneous, which are clearly found and written by AI. I don't have the capacity to review all of them.
|
|
||||||
|
|||||||
+1
-36
@@ -48,8 +48,6 @@ type Agent struct {
|
|||||||
keys []gossh.PublicKey // SSH public keys
|
keys []gossh.PublicKey // SSH public keys
|
||||||
smartManager *SmartManager // Manages SMART data
|
smartManager *SmartManager // Manages SMART data
|
||||||
systemdManager *systemdManager // Manages systemd services
|
systemdManager *systemdManager // Manages systemd services
|
||||||
monitorManager *MonitorManager // Manages network monitors
|
|
||||||
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewAgent creates a new agent with the given data directory for persisting data.
|
// NewAgent creates a new agent with the given data directory for persisting data.
|
||||||
@@ -123,22 +121,6 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
|
|||||||
// initialize handler registry
|
// initialize handler registry
|
||||||
agent.handlerRegistry = NewHandlerRegistry()
|
agent.handlerRegistry = NewHandlerRegistry()
|
||||||
|
|
||||||
// initialize monitor manager
|
|
||||||
agent.monitorManager = newMonitorManager()
|
|
||||||
|
|
||||||
agent.storagePoolManager = newStoragePoolManager()
|
|
||||||
|
|
||||||
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
|
|
||||||
if zfsIntervalEnv, exists := utils.GetEnv("ZFS_INTERVAL"); exists {
|
|
||||||
if duration, err := time.ParseDuration(zfsIntervalEnv); err == nil && duration > 0 {
|
|
||||||
agent.storagePoolManager.detailInterval = duration
|
|
||||||
agent.systemDetails.ZfsInterval = duration
|
|
||||||
slog.Info("ZFS_INTERVAL", "duration", duration)
|
|
||||||
} else {
|
|
||||||
slog.Warn("Invalid ZFS_INTERVAL", "err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// initialize disk info
|
// initialize disk info
|
||||||
agent.initializeDiskInfo()
|
agent.initializeDiskInfo()
|
||||||
|
|
||||||
@@ -196,11 +178,6 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if a.monitorManager != nil {
|
|
||||||
data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
|
|
||||||
slog.Debug("Monitors", "data", data.Monitors)
|
|
||||||
}
|
|
||||||
|
|
||||||
// skip updating systemd services if cache time is not the default 60sec interval
|
// skip updating systemd services if cache time is not the default 60sec interval
|
||||||
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
|
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
|
||||||
totalCount := uint16(a.systemdManager.getServiceStatsCount())
|
totalCount := uint16(a.systemdManager.getServiceStatsCount())
|
||||||
@@ -210,25 +187,13 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
|
|||||||
}
|
}
|
||||||
if a.systemdManager.hasFreshStats {
|
if a.systemdManager.hasFreshStats {
|
||||||
data.SystemdServices = a.systemdManager.getServiceStats(nil, false)
|
data.SystemdServices = a.systemdManager.getServiceStats(nil, false)
|
||||||
data.SystemdServicesUpdated = true
|
|
||||||
// Preserve an explicit zero count so the hub can distinguish a fresh
|
|
||||||
// empty snapshot from a response that omitted systemd data.
|
|
||||||
if totalCount == 0 {
|
|
||||||
data.Info.Services = []uint16{0, 0}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
data.Stats.ExtraFs = make(map[string]*system.FsStats)
|
data.Stats.ExtraFs = make(map[string]*system.FsStats)
|
||||||
data.Info.ExtraFsPct = make(map[string]float64)
|
data.Info.ExtraFsPct = make(map[string]float64)
|
||||||
for name, stats := range a.fsStats {
|
for name, stats := range a.fsStats {
|
||||||
if stats.Root {
|
if !stats.Root && stats.DiskTotal > 0 {
|
||||||
if stats.Name != "" {
|
|
||||||
data.Info.RootDiskName = stats.Name
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if stats.DiskTotal > 0 {
|
|
||||||
// Use custom name if available, otherwise use device name
|
// Use custom name if available, otherwise use device name
|
||||||
key := name
|
key := name
|
||||||
if stats.Name != "" {
|
if stats.Name != "" {
|
||||||
|
|||||||
@@ -33,10 +33,7 @@ var errNoBatteries = errors.New("no readable batteries")
|
|||||||
func normalizeBatteries(batteries []Battery) []Battery {
|
func normalizeBatteries(batteries []Battery) []Battery {
|
||||||
nameCounts := make(map[string]int, len(batteries))
|
nameCounts := make(map[string]int, len(batteries))
|
||||||
for i := range batteries {
|
for i := range batteries {
|
||||||
// Names come from firmware (e.g. sysfs model_name) and are not guaranteed to
|
name := strings.TrimSpace(batteries[i].Name)
|
||||||
// be valid UTF-8. Invalid bytes are rejected when the hub decodes the CBOR
|
|
||||||
// payload, which drops every metric for the system, so strip them here.
|
|
||||||
name := strings.TrimSpace(strings.ToValidUTF8(batteries[i].Name, ""))
|
|
||||||
if name == "" {
|
if name == "" {
|
||||||
name = "Battery " + strconv.Itoa(i+1)
|
name = "Battery " + strconv.Itoa(i+1)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package battery
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -34,15 +33,3 @@ func TestNormalizeBatteriesFallbackNames(t *testing.T) {
|
|||||||
bats := normalizeBatteries([]Battery{{}, {}, {Name: "Mouse"}, {Name: "Mouse"}})
|
bats := normalizeBatteries([]Battery{{}, {}, {Name: "Mouse"}, {Name: "Mouse"}})
|
||||||
assert.Equal(t, []string{"Battery 1", "Battery 2", "Mouse", "Mouse (2)"}, []string{bats[0].Name, bats[1].Name, bats[2].Name, bats[3].Name})
|
assert.Equal(t, []string{"Battery 1", "Battery 2", "Mouse", "Mouse (2)"}, []string{bats[0].Name, bats[1].Name, bats[2].Name, bats[3].Name})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizeBatteriesStripsInvalidUTF8(t *testing.T) {
|
|
||||||
// Firmware occasionally reports names that are not valid UTF-8 (a ThinkPad
|
|
||||||
// reporting "LNV-5B11K63024@\xd0" in model_name is a real example).
|
|
||||||
bats := normalizeBatteries([]Battery{{Name: "LNV-5B11K63024@\xd0"}, {Name: "\xff\xfe"}})
|
|
||||||
assert.Equal(t, "LNV-5B11K63024@", bats[0].Name)
|
|
||||||
// A name made up entirely of invalid bytes falls back to the generic name.
|
|
||||||
assert.Equal(t, "Battery 2", bats[1].Name)
|
|
||||||
for _, b := range bats {
|
|
||||||
assert.True(t, utf8.ValidString(b.Name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,26 +0,0 @@
|
|||||||
// Package btrfs reads btrfs filesystem state from sysfs.
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
// Filesystem is a mounted btrfs filesystem read from /sys/fs/btrfs/<uuid>.
|
|
||||||
type Filesystem struct {
|
|
||||||
UUID string // stable filesystem UUID from sysfs
|
|
||||||
MountID string // kernel filesystem identity for matching monitored mounts
|
|
||||||
IODevice string // sole member block-device name, empty for multi-device/unknown pools
|
|
||||||
Name string // label, else first mountpoint, else UUID
|
|
||||||
Size uint64 // effective usable capacity, or raw member capacity when Raw
|
|
||||||
Raw bool // capacity and usage are physical bytes, unsuitable for disk alerts
|
|
||||||
Alloc uint64 // raw bytes allocated to data, metadata and system chunks
|
|
||||||
Health string // ONLINE, or DEGRADED when a device is missing
|
|
||||||
NRead uint64 // cumulative bytes read across member devices
|
|
||||||
NWrite uint64 // cumulative bytes written across member devices
|
|
||||||
Devices []Device
|
|
||||||
}
|
|
||||||
|
|
||||||
// Device is one member device (devinfo/<devid>) with its error counters.
|
|
||||||
type Device struct {
|
|
||||||
Name string // "devid N"; sysfs does not expose the block device path
|
|
||||||
State string // ONLINE or MISSING
|
|
||||||
ReadErrs uint64
|
|
||||||
WriteErrs uint64
|
|
||||||
CorruptionErrs uint64
|
|
||||||
}
|
|
||||||
@@ -1,285 +0,0 @@
|
|||||||
//go:build linux
|
|
||||||
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/utils"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
sysfsPath = "/sys/fs/btrfs"
|
|
||||||
mountsPath = "/proc/self/mounts"
|
|
||||||
mountinfoPath = "/proc/self/mountinfo"
|
|
||||||
mountUUID = MountID
|
|
||||||
deviceSize = ioctlDeviceSize
|
|
||||||
filesystemUsage = statfsUsage
|
|
||||||
)
|
|
||||||
|
|
||||||
// Filesystems returns all mounted btrfs filesystems, or nil when there are none.
|
|
||||||
func Filesystems() ([]Filesystem, error) {
|
|
||||||
entries, err := os.ReadDir(sysfsPath)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
mounts := mountpointsByDevice()
|
|
||||||
var filesystems []Filesystem
|
|
||||||
for _, entry := range entries {
|
|
||||||
if !entry.IsDir() || entry.Name() == "features" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fs, err := readFilesystem(filepath.Join(sysfsPath, entry.Name()), mounts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("btrfs %s: %w", entry.Name(), err)
|
|
||||||
}
|
|
||||||
filesystems = append(filesystems, fs)
|
|
||||||
}
|
|
||||||
return filesystems, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func readFilesystem(dir string, mounts map[string]string) (Filesystem, error) {
|
|
||||||
fs := Filesystem{UUID: filepath.Base(dir), Name: utils.ReadStringFile(filepath.Join(dir, "label")), Health: "UNKNOWN"}
|
|
||||||
for _, kind := range []string{"data", "metadata", "system"} {
|
|
||||||
if value, ok := utils.ReadUintFile(filepath.Join(dir, "allocation", kind, "disk_used")); ok {
|
|
||||||
fs.Alloc += value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// devices/<name> links to the block device's sysfs directory.
|
|
||||||
devices, err := os.ReadDir(filepath.Join(dir, "devices"))
|
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fs, err
|
|
||||||
}
|
|
||||||
mountpoint := mounts["uuid:"+fs.UUID]
|
|
||||||
if fs.Name == "" {
|
|
||||||
fs.Name = mountpoint
|
|
||||||
}
|
|
||||||
var backingSize uint64
|
|
||||||
for _, dev := range devices {
|
|
||||||
if mountpoint == "" {
|
|
||||||
mountpoint = mounts[dev.Name()]
|
|
||||||
}
|
|
||||||
if fs.Name == "" {
|
|
||||||
fs.Name = mountpoint
|
|
||||||
}
|
|
||||||
devDir := filepath.Join(dir, "devices", dev.Name())
|
|
||||||
if size, ok := utils.ReadUintFile(filepath.Join(devDir, "size")); ok {
|
|
||||||
backingSize += size * 512
|
|
||||||
}
|
|
||||||
if stat := strings.Fields(utils.ReadStringFile(filepath.Join(devDir, "stat"))); len(stat) >= 7 {
|
|
||||||
fs.NRead += parseUint(stat[2]) * 512
|
|
||||||
fs.NWrite += parseUint(stat[6]) * 512
|
|
||||||
}
|
|
||||||
}
|
|
||||||
devids, err := os.ReadDir(filepath.Join(dir, "devinfo"))
|
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fs, err
|
|
||||||
}
|
|
||||||
capacityAvailable := len(devids) > 0
|
|
||||||
healthKnown := len(devids) > 0
|
|
||||||
for _, devid := range devids {
|
|
||||||
devDir := filepath.Join(dir, "devinfo", devid.Name())
|
|
||||||
// Replacement targets do not add filesystem capacity.
|
|
||||||
replaceTarget, _ := utils.ReadUintFile(filepath.Join(devDir, "replace_target"))
|
|
||||||
if replaceTarget != 1 {
|
|
||||||
devid, err := strconv.ParseUint(devid.Name(), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return fs, err
|
|
||||||
}
|
|
||||||
size, err := deviceSize(mountpoint, devid)
|
|
||||||
if err != nil {
|
|
||||||
capacityAvailable = false
|
|
||||||
}
|
|
||||||
fs.Size += size
|
|
||||||
}
|
|
||||||
dev := Device{Name: "devid " + devid.Name(), State: "ONLINE"}
|
|
||||||
missing := utils.ReadStringFile(filepath.Join(devDir, "missing"))
|
|
||||||
if missing != "0" && missing != "1" {
|
|
||||||
healthKnown = false
|
|
||||||
dev.State = "UNKNOWN"
|
|
||||||
}
|
|
||||||
if missing == "1" {
|
|
||||||
dev.State = "MISSING"
|
|
||||||
fs.Health = "DEGRADED"
|
|
||||||
}
|
|
||||||
for line := range strings.Lines(utils.ReadStringFile(filepath.Join(devDir, "error_stats"))) {
|
|
||||||
if fields := strings.Fields(line); len(fields) == 2 {
|
|
||||||
switch fields[0] {
|
|
||||||
case "read_errs":
|
|
||||||
dev.ReadErrs = parseUint(fields[1])
|
|
||||||
case "write_errs":
|
|
||||||
dev.WriteErrs = parseUint(fields[1])
|
|
||||||
case "corruption_errs":
|
|
||||||
dev.CorruptionErrs = parseUint(fields[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fs.Devices = append(fs.Devices, dev)
|
|
||||||
}
|
|
||||||
// Use one capacity source for the whole filesystem: device IDs cannot be
|
|
||||||
// reliably matched to block-device names in sysfs. A partial ioctl result
|
|
||||||
// must not be added to the complete backing-device total.
|
|
||||||
if !capacityAvailable {
|
|
||||||
fs.Size = backingSize
|
|
||||||
}
|
|
||||||
if fs.Health != "DEGRADED" && healthKnown {
|
|
||||||
fs.Health = "ONLINE"
|
|
||||||
}
|
|
||||||
fs.MountID = mountUUID(mountpoint)
|
|
||||||
if len(devices) == 1 && len(devids) == 1 && fs.Health == "ONLINE" {
|
|
||||||
fs.IODevice = devices[0].Name()
|
|
||||||
}
|
|
||||||
fs.Raw = true
|
|
||||||
if used, available, err := filesystemUsage(mountpoint); err == nil {
|
|
||||||
// Effective capacity excludes reserved/unavailable space, so Size-Alloc
|
|
||||||
// is available to applications and the usage ratio matches df.
|
|
||||||
fs.Size, fs.Alloc, fs.Raw = used+available, used, false
|
|
||||||
}
|
|
||||||
if fs.Name == "" {
|
|
||||||
fs.Name = filepath.Base(dir)
|
|
||||||
}
|
|
||||||
return fs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mountpointsByDevice prefers UUID matches from mountinfo and retains source
|
|
||||||
// device names as a fallback for environments where FS_INFO is unavailable.
|
|
||||||
func mountpointsByDevice() map[string]string {
|
|
||||||
mounts := mountpointsByUUID(utils.ReadStringFile(mountinfoPath), mountUUID)
|
|
||||||
for line := range strings.Lines(utils.ReadStringFile(mountsPath)) {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) < 3 || fields[2] != "btrfs" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
device := fields[0]
|
|
||||||
if resolved, err := filepath.EvalSymlinks(device); err == nil {
|
|
||||||
device = resolved
|
|
||||||
}
|
|
||||||
if _, seen := mounts[filepath.Base(device)]; !seen {
|
|
||||||
mounts[filepath.Base(device)] = unescapeMountPath(fields[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return mounts
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseUint(s string) uint64 {
|
|
||||||
n, _ := strconv.ParseUint(s, 10, 64)
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
// ioctlDeviceSize reads Btrfs's recorded device size, which can be smaller
|
|
||||||
// than the block device after a filesystem resize. BTRFS_IOC_DEV_INFO is
|
|
||||||
// _IOWR(0x94, 30, struct btrfs_ioctl_dev_info_args), a 4096-byte ABI structure.
|
|
||||||
func ioctlDeviceSize(mountpoint string, devid uint64) (uint64, error) {
|
|
||||||
if mountpoint == "" {
|
|
||||||
return 0, errors.New("no accessible mountpoint")
|
|
||||||
}
|
|
||||||
f, err := os.Open(mountpoint)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
args := struct {
|
|
||||||
Devid uint64
|
|
||||||
UUID [16]byte
|
|
||||||
BytesUsed uint64
|
|
||||||
TotalBytes uint64
|
|
||||||
Reserved [4096 - 40]byte
|
|
||||||
}{Devid: devid}
|
|
||||||
_, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), 0xd000941e, uintptr(unsafe.Pointer(&args)))
|
|
||||||
if errno != 0 {
|
|
||||||
return 0, errno
|
|
||||||
}
|
|
||||||
return args.TotalBytes, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The filesystem magic is unsigned even when Statfs_t.Type is int32.
|
|
||||||
func isBtrfs(stat *unix.Statfs_t) bool {
|
|
||||||
return uint32(stat.Type) == unix.BTRFS_SUPER_MAGIC
|
|
||||||
}
|
|
||||||
|
|
||||||
func statfsUsage(path string) (used, available uint64, err error) {
|
|
||||||
if path == "" {
|
|
||||||
return 0, 0, errors.New("no accessible mountpoint")
|
|
||||||
}
|
|
||||||
var stat unix.Statfs_t
|
|
||||||
if err = unix.Statfs(path, &stat); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !isBtrfs(&stat) {
|
|
||||||
return 0, 0, errors.New("mountpoint is not Btrfs")
|
|
||||||
}
|
|
||||||
blockSize := uint64(stat.Bsize)
|
|
||||||
return (stat.Blocks - min(stat.Blocks, stat.Bfree)) * blockSize, min(stat.Blocks, stat.Bavail) * blockSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// MountID returns the filesystem UUID via BTRFS_IOC_FS_INFO. Unlike statfs
|
|
||||||
// f_fsid, this identity is shared by all subvolumes and bind mounts.
|
|
||||||
func MountID(path string) string {
|
|
||||||
if path == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
var stat unix.Statfs_t
|
|
||||||
if unix.Statfs(path, &stat) != nil || !isBtrfs(&stat) {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
f, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
args := struct {
|
|
||||||
MaxID uint64
|
|
||||||
NumDevices uint64
|
|
||||||
FSID [16]byte
|
|
||||||
Reserved [992]byte
|
|
||||||
}{}
|
|
||||||
// _IOR(0x94, 31, 1024). Reuse the platform's read-direction bits;
|
|
||||||
// MIPS/PowerPC use a different encoding than asm-generic.
|
|
||||||
request := uintptr(unix.FS_IOC_GETFLAGS&0xe0000000) | 0x0400941f
|
|
||||||
_, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), request, uintptr(unsafe.Pointer(&args)))
|
|
||||||
if errno != 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
id := args.FSID
|
|
||||||
return fmt.Sprintf("%x-%x-%x-%x-%x", id[:4], id[4:6], id[6:8], id[8:10], id[10:])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Btrfs mountinfo device numbers can be virtual (0:N), so query the UUID
|
|
||||||
// through the mount instead of comparing those numbers with sysfs block devs.
|
|
||||||
// Retry another path when a bind mount is inaccessible. Once resolved, reuse
|
|
||||||
// the result for that mount device to avoid opening every Docker bind mount.
|
|
||||||
func mountpointsByUUID(mountinfo string, identify func(string) string) map[string]string {
|
|
||||||
mounts := make(map[string]string)
|
|
||||||
resolved := make(map[string]bool)
|
|
||||||
for line := range strings.Lines(mountinfo) {
|
|
||||||
before, after, ok := strings.Cut(line, " - ")
|
|
||||||
fields, fs := strings.Fields(before), strings.Fields(after)
|
|
||||||
if !ok || len(fields) < 6 || len(fs) < 3 || fs[0] != "btrfs" || resolved[fields[2]] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
path := unescapeMountPath(fields[4])
|
|
||||||
uuid := identify(path)
|
|
||||||
if uuid == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
resolved[fields[2]] = true
|
|
||||||
if mounts["uuid:"+uuid] == "" {
|
|
||||||
mounts["uuid:"+uuid] = path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return mounts
|
|
||||||
}
|
|
||||||
|
|
||||||
func unescapeMountPath(path string) string {
|
|
||||||
return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(path)
|
|
||||||
}
|
|
||||||
@@ -1,274 +0,0 @@
|
|||||||
//go:build testing && linux
|
|
||||||
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/utils"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFilesystems(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldSysfs, oldMounts := sysfsPath, mountsPath
|
|
||||||
sysfsPath, mountsPath = root, filepath.Join(root, "mounts")
|
|
||||||
t.Cleanup(func() { sysfsPath, mountsPath = oldSysfs, oldMounts })
|
|
||||||
|
|
||||||
fsDir := filepath.Join(root, "1b2c3d4e-0000-0000-0000-000000000000")
|
|
||||||
write := func(rel, content string) {
|
|
||||||
path := filepath.Join(fsDir, rel)
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
|
|
||||||
}
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "features"), 0o755))
|
|
||||||
oldUsage := filesystemUsage
|
|
||||||
filesystemUsage = func(string) (uint64, uint64, error) { return 0, 0, os.ErrNotExist }
|
|
||||||
t.Cleanup(func() { filesystemUsage = oldUsage })
|
|
||||||
oldDeviceSize := deviceSize
|
|
||||||
t.Cleanup(func() { deviceSize = oldDeviceSize })
|
|
||||||
deviceSize = func(_ string, devid uint64) (uint64, error) {
|
|
||||||
value, _ := utils.ReadUintFile(filepath.Join(fsDir, "recorded-size", strconv.FormatUint(devid, 10)))
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
// Recorded member capacities differ from the unchanged backing devices.
|
|
||||||
write("recorded-size/1", "256000\n")
|
|
||||||
write("recorded-size/2", "128000\n")
|
|
||||||
write("label", "tank\n")
|
|
||||||
write("allocation/data/disk_used", "4096\n")
|
|
||||||
write("allocation/metadata/disk_used", "2048\n")
|
|
||||||
write("allocation/system/disk_used", "1024\n")
|
|
||||||
write("devices/sda/size", "1000\n")
|
|
||||||
write("devices/sda/stat", "10 0 200 0 20 0 400 0 0 0 0\n")
|
|
||||||
write("devices/sdb/size", "1000\n")
|
|
||||||
write("devices/sdb/stat", "10 0 100 0 20 0 100 0 0 0 0\n")
|
|
||||||
write("devinfo/1/missing", "0\n")
|
|
||||||
write("devinfo/1/error_stats", "write_errs 1\nread_errs 2\nflush_errs 0\ncorruption_errs 3\ngeneration_errs 0\n")
|
|
||||||
write("devinfo/2/missing", "1\n")
|
|
||||||
|
|
||||||
filesystems, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, filesystems, 1)
|
|
||||||
assert.Equal(t, Filesystem{
|
|
||||||
UUID: "1b2c3d4e-0000-0000-0000-000000000000", Raw: true, Name: "tank", Size: 384000, Alloc: 7168, Health: "DEGRADED", NRead: 153600, NWrite: 256000,
|
|
||||||
Devices: []Device{
|
|
||||||
{Name: "devid 1", State: "ONLINE", ReadErrs: 2, WriteErrs: 1, CorruptionErrs: 3},
|
|
||||||
{Name: "devid 2", State: "MISSING"},
|
|
||||||
},
|
|
||||||
}, filesystems[0])
|
|
||||||
|
|
||||||
// Unlabeled filesystems fall back to the first mountpoint, then the UUID.
|
|
||||||
write("label", "\n")
|
|
||||||
require.NoError(t, os.WriteFile(mountsPath, []byte(
|
|
||||||
"/dev/sdz1 /other btrfs rw 0 0\n/dev/sdb /mnt/storage btrfs rw 0 0\n/dev/sdb /mnt/storage/sub btrfs rw,subvol=/sub 0 0\n",
|
|
||||||
), 0o644))
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "/mnt/storage", filesystems[0].Name)
|
|
||||||
|
|
||||||
require.NoError(t, os.Remove(mountsPath))
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "1b2c3d4e-0000-0000-0000-000000000000", filesystems[0].Name)
|
|
||||||
write("devinfo/3/replace_target", "1\n")
|
|
||||||
write("recorded-size/3", "512000\n")
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(384000), filesystems[0].Size, "replacement target must not inflate capacity")
|
|
||||||
|
|
||||||
deviceSize = func(string, uint64) (uint64, error) { return 0, os.ErrPermission }
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, filesystems, 1)
|
|
||||||
assert.Equal(t, uint64(1024000), filesystems[0].Size)
|
|
||||||
assert.Equal(t, "DEGRADED", filesystems[0].Health)
|
|
||||||
assert.Equal(t, uint64(153600), filesystems[0].NRead)
|
|
||||||
|
|
||||||
// A partial ioctl result must not be mixed with the backing-device total.
|
|
||||||
deviceSize = func(_ string, devid uint64) (uint64, error) {
|
|
||||||
if devid == 2 {
|
|
||||||
return 0, os.ErrPermission
|
|
||||||
}
|
|
||||||
return 256000, nil
|
|
||||||
}
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(1024000), filesystems[0].Size)
|
|
||||||
|
|
||||||
// With no mount visible (e.g. Docker), the real lookup falls back too.
|
|
||||||
deviceSize = ioctlDeviceSize
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, filesystems, 1)
|
|
||||||
assert.Equal(t, uint64(1024000), filesystems[0].Size)
|
|
||||||
|
|
||||||
filesystemUsage = func(string) (uint64, uint64, error) { return 100, 900, nil }
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(1000), filesystems[0].Size)
|
|
||||||
assert.Equal(t, uint64(100), filesystems[0].Alloc)
|
|
||||||
assert.False(t, filesystems[0].Raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFilesystemsNoBtrfs(t *testing.T) {
|
|
||||||
oldPath := sysfsPath
|
|
||||||
sysfsPath = filepath.Join(t.TempDir(), "missing")
|
|
||||||
t.Cleanup(func() { sysfsPath = oldPath })
|
|
||||||
|
|
||||||
filesystems, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Nil(t, filesystems)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIoctlDeviceSizeFailure(t *testing.T) {
|
|
||||||
_, err := ioctlDeviceSize("", 1)
|
|
||||||
require.Error(t, err)
|
|
||||||
_, err = ioctlDeviceSize(t.TempDir(), 1)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.ErrorIs(t, err, unix.ENOTTY)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMountpointsDecodeEscapes(t *testing.T) {
|
|
||||||
oldMounts := mountsPath
|
|
||||||
mountsPath = filepath.Join(t.TempDir(), "mounts")
|
|
||||||
t.Cleanup(func() { mountsPath = oldMounts })
|
|
||||||
require.NoError(t, os.WriteFile(mountsPath, []byte("/dev/test-btrfs /mnt/my\\040data btrfs rw 0 0\n"), 0o644))
|
|
||||||
assert.Equal(t, "/mnt/my data", mountpointsByDevice()["test-btrfs"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFilesystemWithoutDevinfo(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "devices", "sda"), 0755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(root, "devices", "sda", "size"), []byte("1000"), 0644))
|
|
||||||
fs, err := readFilesystem(root, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(512000), fs.Size)
|
|
||||||
assert.True(t, fs.Raw)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Health)
|
|
||||||
assert.Empty(t, fs.Devices)
|
|
||||||
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "devinfo", "1"), 0755))
|
|
||||||
fs, err = readFilesystem(root, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Health)
|
|
||||||
require.Len(t, fs.Devices, 1)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Devices[0].State)
|
|
||||||
|
|
||||||
// Some older interfaces lack the devices directory too.
|
|
||||||
fs, err = readFilesystem(t.TempDir(), nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Health)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalBtrfsUsage(t *testing.T) {
|
|
||||||
path := os.Getenv("BESZEL_TEST_BTRFS_MOUNT")
|
|
||||||
if path == "" {
|
|
||||||
t.Skip("set BESZEL_TEST_BTRFS_MOUNT for read-only live validation")
|
|
||||||
}
|
|
||||||
used, available, err := statfsUsage(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
filesystems, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, fs := range filesystems {
|
|
||||||
if !fs.Raw && fs.Alloc == used && fs.Size == used+available {
|
|
||||||
t.Logf("pool=%s used=%d available=%d effective_capacity=%d", fs.Name, used, available, fs.Size)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatal("collector did not report the mounted filesystem's usable capacity")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMountID(t *testing.T) {
|
|
||||||
assert.Empty(t, MountID(""))
|
|
||||||
assert.Empty(t, MountID(filepath.Join(t.TempDir(), "missing")))
|
|
||||||
path := os.Getenv("BESZEL_TEST_BTRFS_MOUNT")
|
|
||||||
if path == "" {
|
|
||||||
t.Skip("set BESZEL_TEST_BTRFS_MOUNT for live identity validation")
|
|
||||||
}
|
|
||||||
id := MountID(path)
|
|
||||||
require.NotEmpty(t, id)
|
|
||||||
assert.Equal(t, id, MountID(filepath.Join(path, ".")))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMountinfoUUIDLookup(t *testing.T) {
|
|
||||||
info := `1 0 0:40 /@ /inaccessible ro shared:1 - btrfs /dev/mapper/unavailable rw
|
|
||||||
2 0 0:40 /@/docker/hosts /etc/hosts ro - btrfs /dev/mapper/unavailable rw
|
|
||||||
3 0 0:40 /@/docker/hostname /etc/hostname ro - btrfs /dev/mapper/unavailable rw
|
|
||||||
4 0 0:41 /subvol /extra-filesystems/my\040disk ro master:2 - btrfs /dev/missing rw
|
|
||||||
5 0 0:42 / /ext4 ro - ext4 /dev/mapper/unavailable rw
|
|
||||||
malformed
|
|
||||||
6 0 0:43 / /bad ro - btrfs
|
|
||||||
`
|
|
||||||
var calls []string
|
|
||||||
mounts := mountpointsByUUID(info, func(path string) string {
|
|
||||||
calls = append(calls, path)
|
|
||||||
switch path {
|
|
||||||
case "/etc/hosts":
|
|
||||||
return "root-uuid"
|
|
||||||
case "/extra-filesystems/my disk":
|
|
||||||
return "extra-uuid"
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
})
|
|
||||||
assert.Equal(t, map[string]string{"uuid:root-uuid": "/etc/hosts", "uuid:extra-uuid": "/extra-filesystems/my disk"}, mounts)
|
|
||||||
assert.Equal(t, []string{"/inaccessible", "/etc/hosts", "/extra-filesystems/my disk"}, calls)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDockerFilesystemWithoutDeviceNodes(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldSysfs, oldMounts, oldInfo, oldUUID, oldUsage := sysfsPath, mountsPath, mountinfoPath, mountUUID, filesystemUsage
|
|
||||||
t.Cleanup(func() {
|
|
||||||
sysfsPath, mountsPath, mountinfoPath, mountUUID, filesystemUsage = oldSysfs, oldMounts, oldInfo, oldUUID, oldUsage
|
|
||||||
})
|
|
||||||
sysfsPath = filepath.Join(root, "sysfs")
|
|
||||||
mountsPath = filepath.Join(root, "missing-mounts")
|
|
||||||
mountinfoPath = filepath.Join(root, "mountinfo")
|
|
||||||
uuid := "11111111-1111-4111-8111-111111111111"
|
|
||||||
dir := filepath.Join(sysfsPath, uuid)
|
|
||||||
for path, content := range map[string]string{"devices/dm-0/size": "1000", "devinfo/1/missing": "0"} {
|
|
||||||
target := filepath.Join(dir, path)
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0755))
|
|
||||||
require.NoError(t, os.WriteFile(target, []byte(content), 0644))
|
|
||||||
}
|
|
||||||
require.NoError(t, os.WriteFile(mountinfoPath, []byte("2 1 0:40 /@/docker/hosts /etc/hosts ro - btrfs /dev/mapper/not-in-container rw\n"), 0644))
|
|
||||||
mountUUID = func(path string) string {
|
|
||||||
if path == "/etc/hosts" {
|
|
||||||
return uuid
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
filesystemUsage = func(path string) (uint64, uint64, error) { require.Equal(t, "/etc/hosts", path); return 100, 900, nil }
|
|
||||||
fs, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, fs, 1)
|
|
||||||
assert.Equal(t, uuid, fs[0].MountID)
|
|
||||||
assert.Equal(t, "dm-0", fs[0].IODevice)
|
|
||||||
assert.False(t, fs[0].Raw)
|
|
||||||
assert.Equal(t, uint64(1000), fs[0].Size)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLivePoolMountIdentity(t *testing.T) {
|
|
||||||
path := os.Getenv("BESZEL_TEST_BTRFS_MOUNT")
|
|
||||||
if path == "" {
|
|
||||||
t.Skip("set BESZEL_TEST_BTRFS_MOUNT for live validation")
|
|
||||||
}
|
|
||||||
id := MountID(path)
|
|
||||||
require.NotEmpty(t, id)
|
|
||||||
pools, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, pool := range pools {
|
|
||||||
if pool.UUID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
assert.Equal(t, id, pool.MountID)
|
|
||||||
assert.False(t, pool.Raw)
|
|
||||||
t.Logf("uuid=%s mount_identity=%s io_device=%s raw=%v", pool.UUID, pool.MountID, pool.IODevice, pool.Raw)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Fatal("mounted Btrfs filesystem was not discovered")
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
//go:build !linux
|
|
||||||
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
import "errors"
|
|
||||||
|
|
||||||
func Filesystems() ([]Filesystem, error) {
|
|
||||||
return nil, errors.ErrUnsupported
|
|
||||||
}
|
|
||||||
|
|
||||||
func MountID(string) string { return "" }
|
|
||||||
+4
-74
@@ -2,7 +2,6 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"crypto/x509"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -25,28 +24,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// Keep the connection alive long enough for a slow collection cycle to
|
wsDeadline = 70 * time.Second
|
||||||
// finish before the hub considers the agent disconnected.
|
|
||||||
wsDeadline = 120 * time.Second
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// errNoHubURL is returned when HUB_URL is unset. This is not a failure
|
|
||||||
// condition: an agent configured with only a public key runs in SSH-only mode,
|
|
||||||
// where the hub dials the agent and no outbound WebSocket client is expected.
|
|
||||||
var errNoHubURL = errors.New("HUB_URL environment variable not set")
|
|
||||||
|
|
||||||
type caCertFileError struct {
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *caCertFileError) Error() string {
|
|
||||||
return e.err.Error()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *caCertFileError) Unwrap() error {
|
|
||||||
return e.err
|
|
||||||
}
|
|
||||||
|
|
||||||
// WebSocketClient manages the WebSocket connection between the agent and hub.
|
// WebSocketClient manages the WebSocket connection between the agent and hub.
|
||||||
// It handles authentication, message routing, and connection lifecycle management.
|
// It handles authentication, message routing, and connection lifecycle management.
|
||||||
type WebSocketClient struct {
|
type WebSocketClient struct {
|
||||||
@@ -60,7 +40,6 @@ type WebSocketClient struct {
|
|||||||
hubRequest *common.HubRequest[cbor.RawMessage] // Reusable request structure for message parsing
|
hubRequest *common.HubRequest[cbor.RawMessage] // Reusable request structure for message parsing
|
||||||
lastConnectAttempt time.Time // Timestamp of last connection attempt
|
lastConnectAttempt time.Time // Timestamp of last connection attempt
|
||||||
hubVerified bool // Whether the hub has been cryptographically verified
|
hubVerified bool // Whether the hub has been cryptographically verified
|
||||||
tlsConfig *tls.Config // Optional TLS configuration with custom CA certificates
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWebSocketClient creates a new WebSocket client for the given agent.
|
// newWebSocketClient creates a new WebSocket client for the given agent.
|
||||||
@@ -68,7 +47,7 @@ type WebSocketClient struct {
|
|||||||
func newWebSocketClient(agent *Agent) (client *WebSocketClient, err error) {
|
func newWebSocketClient(agent *Agent) (client *WebSocketClient, err error) {
|
||||||
hubURLStr, exists := utils.GetEnv("HUB_URL")
|
hubURLStr, exists := utils.GetEnv("HUB_URL")
|
||||||
if !exists {
|
if !exists {
|
||||||
return nil, errNoHubURL
|
return nil, errors.New("HUB_URL environment variable not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
client = &WebSocketClient{}
|
client = &WebSocketClient{}
|
||||||
@@ -82,10 +61,6 @@ func newWebSocketClient(agent *Agent) (client *WebSocketClient, err error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
client.tlsConfig, err = getTLSConfig()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
client.agent = agent
|
client.agent = agent
|
||||||
client.hubRequest = &common.HubRequest[cbor.RawMessage]{}
|
client.hubRequest = &common.HubRequest[cbor.RawMessage]{}
|
||||||
@@ -112,52 +87,7 @@ func getToken() (string, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return parseTokenFile(string(tokenBytes), tokenFile)
|
return strings.TrimSpace(string(tokenBytes)), nil
|
||||||
}
|
|
||||||
|
|
||||||
// parseTokenFile reads a single token from TOKEN_FILE.
|
|
||||||
// Blank lines and comments are ignored. Multiple tokens are rejected because
|
|
||||||
// the agent supports only one outbound hub connection.
|
|
||||||
func parseTokenFile(contents, path string) (string, error) {
|
|
||||||
var token string
|
|
||||||
for line := range strings.Lines(contents) {
|
|
||||||
line = strings.TrimSpace(line)
|
|
||||||
if len(line) == 0 || strings.HasPrefix(line, "#") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if token != "" {
|
|
||||||
return "", fmt.Errorf("%s must contain a single token", path)
|
|
||||||
}
|
|
||||||
token = line
|
|
||||||
}
|
|
||||||
// An empty file keeps returning an empty token, as before: the caller decides
|
|
||||||
// what to do about it.
|
|
||||||
return token, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// getTLSConfig returns a TLS configuration containing the system certificate
|
|
||||||
// pool plus any certificates configured through CA_CERT_FILE. A nil config lets
|
|
||||||
// gws use Go's default TLS configuration and system roots.
|
|
||||||
func getTLSConfig() (*tls.Config, error) {
|
|
||||||
caCertFile, _ := utils.GetEnv("CA_CERT_FILE")
|
|
||||||
if caCertFile == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
caCertPEM, err := os.ReadFile(caCertFile)
|
|
||||||
if err != nil {
|
|
||||||
return nil, &caCertFileError{fmt.Errorf("read CA_CERT_FILE %q: %w", caCertFile, err)}
|
|
||||||
}
|
|
||||||
|
|
||||||
rootCAs, err := x509.SystemCertPool()
|
|
||||||
if err != nil {
|
|
||||||
return nil, &caCertFileError{fmt.Errorf("load system CA certificate pool: %w", err)}
|
|
||||||
}
|
|
||||||
if !rootCAs.AppendCertsFromPEM(caCertPEM) {
|
|
||||||
return nil, &caCertFileError{fmt.Errorf("CA_CERT_FILE %q does not contain any valid PEM certificates", caCertFile)}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &tls.Config{RootCAs: rootCAs}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// getOptions returns the WebSocket client options, creating them if necessary.
|
// getOptions returns the WebSocket client options, creating them if necessary.
|
||||||
@@ -182,7 +112,7 @@ func (client *WebSocketClient) getOptions() *gws.ClientOption {
|
|||||||
|
|
||||||
client.options = &gws.ClientOption{
|
client.options = &gws.ClientOption{
|
||||||
Addr: client.hubURL.String(),
|
Addr: client.hubURL.String(),
|
||||||
TlsConfig: client.tlsConfig,
|
TlsConfig: &tls.Config{InsecureSkipVerify: true},
|
||||||
RequestHeader: http.Header{
|
RequestHeader: http.Header{
|
||||||
"User-Agent": []string{getUserAgent()},
|
"User-Agent": []string{getUserAgent()},
|
||||||
"X-Token": []string{client.token},
|
"X-Token": []string{client.token},
|
||||||
|
|||||||
@@ -4,19 +4,8 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"crypto/rand"
|
|
||||||
"crypto/rsa"
|
|
||||||
"crypto/tls"
|
|
||||||
"crypto/x509"
|
|
||||||
"crypto/x509/pkix"
|
|
||||||
"encoding/pem"
|
|
||||||
"math/big"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -26,34 +15,11 @@ import (
|
|||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
"github.com/fxamacker/cbor/v2"
|
||||||
"github.com/lxzan/gws"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestNewWebSocketClientNoHubURL verifies that an unset HUB_URL returns the
|
|
||||||
// errNoHubURL sentinel rather than an opaque error. Callers rely on this to
|
|
||||||
// distinguish SSH-only mode -- a supported configuration in which the hub dials
|
|
||||||
// the agent -- from an actual misconfiguration.
|
|
||||||
func TestNewWebSocketClientNoHubURL(t *testing.T) {
|
|
||||||
agent := createTestAgent(t)
|
|
||||||
|
|
||||||
// t.Setenv registers restoration of the original value; unset afterwards so
|
|
||||||
// GetEnv's LookupEnv reports the variable as absent rather than empty.
|
|
||||||
t.Setenv("BESZEL_AGENT_HUB_URL", "")
|
|
||||||
os.Unsetenv("BESZEL_AGENT_HUB_URL")
|
|
||||||
t.Setenv("HUB_URL", "")
|
|
||||||
os.Unsetenv("HUB_URL")
|
|
||||||
t.Setenv("BESZEL_AGENT_TOKEN", "test-token")
|
|
||||||
|
|
||||||
client, err := newWebSocketClient(agent)
|
|
||||||
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Nil(t, client)
|
|
||||||
assert.ErrorIs(t, err, errNoHubURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewWebSocketClient tests WebSocket client creation
|
// TestNewWebSocketClient tests WebSocket client creation
|
||||||
func TestNewWebSocketClient(t *testing.T) {
|
func TestNewWebSocketClient(t *testing.T) {
|
||||||
agent := createTestAgent(t)
|
agent := createTestAgent(t)
|
||||||
@@ -198,155 +164,6 @@ func TestWebSocketClient_GetOptions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebSocketClient_TLSVerification(t *testing.T) {
|
|
||||||
agent := createTestAgent(t)
|
|
||||||
serverCert, serverCertPEM := newSelfSignedServerCertificate(t)
|
|
||||||
upgrader := gws.NewUpgrader(&gws.BuiltinEventHandler{}, nil)
|
|
||||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
conn, err := upgrader.Upgrade(w, r)
|
|
||||||
if err == nil {
|
|
||||||
go conn.ReadLoop()
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
server.TLS = &tls.Config{Certificates: []tls.Certificate{serverCert}}
|
|
||||||
server.StartTLS()
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
caCertFile := filepath.Join(t.TempDir(), "hub-ca.crt")
|
|
||||||
require.NoError(t, os.WriteFile(caCertFile, serverCertPEM, 0600))
|
|
||||||
|
|
||||||
newClient := func(t *testing.T, caCertFile string) *WebSocketClient {
|
|
||||||
t.Helper()
|
|
||||||
t.Setenv("BESZEL_AGENT_HUB_URL", server.URL)
|
|
||||||
t.Setenv("BESZEL_AGENT_TOKEN", "test-token")
|
|
||||||
t.Setenv("BESZEL_AGENT_CA_CERT_FILE", caCertFile)
|
|
||||||
client, err := newWebSocketClient(agent)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("system roots are used by default", func(t *testing.T) {
|
|
||||||
client := newClient(t, "")
|
|
||||||
assert.Nil(t, client.getOptions().TlsConfig)
|
|
||||||
_, _, err := gws.NewClient(&gws.BuiltinEventHandler{}, client.getOptions())
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("custom CA trusts self-signed certificate", func(t *testing.T) {
|
|
||||||
systemRoots, err := x509.SystemCertPool()
|
|
||||||
require.NoError(t, err)
|
|
||||||
client := newClient(t, caCertFile)
|
|
||||||
assert.Greater(t, len(client.getOptions().TlsConfig.RootCAs.Subjects()), len(systemRoots.Subjects()))
|
|
||||||
conn, _, err := gws.NewClient(&gws.BuiltinEventHandler{}, client.getOptions())
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, conn.NetConn().Close())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("custom CA does not bypass hostname verification", func(t *testing.T) {
|
|
||||||
client := newClient(t, caCertFile)
|
|
||||||
client.getOptions().TlsConfig.ServerName = "wrong.example.com"
|
|
||||||
_, _, err := gws.NewClient(&gws.BuiltinEventHandler{}, client.getOptions())
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebSocketClient_NonTLSConnection(t *testing.T) {
|
|
||||||
agent := createTestAgent(t)
|
|
||||||
upgrader := gws.NewUpgrader(&gws.BuiltinEventHandler{}, nil)
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
conn, err := upgrader.Upgrade(w, r)
|
|
||||||
if err == nil {
|
|
||||||
go conn.ReadLoop()
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
t.Setenv("BESZEL_AGENT_HUB_URL", server.URL)
|
|
||||||
t.Setenv("BESZEL_AGENT_TOKEN", "test-token")
|
|
||||||
t.Setenv("BESZEL_AGENT_CA_CERT_FILE", "")
|
|
||||||
client, err := newWebSocketClient(agent)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Nil(t, client.getOptions().TlsConfig)
|
|
||||||
|
|
||||||
conn, _, err := gws.NewClient(&gws.BuiltinEventHandler{}, client.getOptions())
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, conn.NetConn().Close())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetTLSConfigErrors(t *testing.T) {
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
testCases := []struct {
|
|
||||||
name string
|
|
||||||
path string
|
|
||||||
contents []byte
|
|
||||||
errorMatch string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "missing file",
|
|
||||||
path: filepath.Join(tempDir, "missing.pem"),
|
|
||||||
errorMatch: "read CA_CERT_FILE",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unreadable path",
|
|
||||||
path: tempDir,
|
|
||||||
errorMatch: "read CA_CERT_FILE",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty file",
|
|
||||||
path: filepath.Join(tempDir, "empty.pem"),
|
|
||||||
contents: []byte{},
|
|
||||||
errorMatch: "does not contain any valid PEM certificates",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "malformed file",
|
|
||||||
path: filepath.Join(tempDir, "malformed.pem"),
|
|
||||||
contents: []byte("not a PEM certificate"),
|
|
||||||
errorMatch: "does not contain any valid PEM certificates",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if tc.contents != nil {
|
|
||||||
require.NoError(t, os.WriteFile(tc.path, tc.contents, 0600))
|
|
||||||
}
|
|
||||||
t.Setenv("BESZEL_AGENT_CA_CERT_FILE", tc.path)
|
|
||||||
|
|
||||||
tlsConfig, err := getTLSConfig()
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Nil(t, tlsConfig)
|
|
||||||
assert.Contains(t, err.Error(), tc.errorMatch)
|
|
||||||
assert.Contains(t, err.Error(), tc.path)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newSelfSignedServerCertificate(t *testing.T) (tls.Certificate, []byte) {
|
|
||||||
t.Helper()
|
|
||||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
template := &x509.Certificate{
|
|
||||||
SerialNumber: big.NewInt(1),
|
|
||||||
Subject: pkix.Name{CommonName: "127.0.0.1"},
|
|
||||||
NotBefore: time.Now().Add(-time.Hour),
|
|
||||||
NotAfter: time.Now().Add(time.Hour),
|
|
||||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
|
||||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
|
|
||||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
||||||
BasicConstraintsValid: true,
|
|
||||||
IsCA: true,
|
|
||||||
}
|
|
||||||
certDER, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
|
|
||||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)})
|
|
||||||
certificate, err := tls.X509KeyPair(certPEM, keyPEM)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return certificate, certPEM
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebSocketClient_VerifySignature tests signature verification
|
// TestWebSocketClient_VerifySignature tests signature verification
|
||||||
func TestWebSocketClient_VerifySignature(t *testing.T) {
|
func TestWebSocketClient_VerifySignature(t *testing.T) {
|
||||||
agent := createTestAgent(t)
|
agent := createTestAgent(t)
|
||||||
@@ -592,41 +409,6 @@ func TestGetToken(t *testing.T) {
|
|||||||
assert.Equal(t, expectedToken, token)
|
assert.Equal(t, expectedToken, token)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("TOKEN_FILE with surrounding blank lines and comments", func(t *testing.T) {
|
|
||||||
expectedToken := "test-token-with-noise"
|
|
||||||
tokenFile := filepath.Join(t.TempDir(), "token")
|
|
||||||
require.NoError(t, os.WriteFile(tokenFile, []byte("# hub token\n\n"+expectedToken+"\n\n"), 0o600))
|
|
||||||
|
|
||||||
t.Setenv("TOKEN_FILE", tokenFile)
|
|
||||||
|
|
||||||
token, err := getToken()
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.Equal(t, expectedToken, token)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("TOKEN_FILE with multiple tokens is rejected", func(t *testing.T) {
|
|
||||||
tokenFile := filepath.Join(t.TempDir(), "token")
|
|
||||||
require.NoError(t, os.WriteFile(tokenFile, []byte("11111111-1111-1111-1111-111111111111\n22222222-2222-2222-2222-222222222222\n"), 0o600))
|
|
||||||
|
|
||||||
t.Setenv("TOKEN_FILE", tokenFile)
|
|
||||||
|
|
||||||
token, err := getToken()
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Empty(t, token)
|
|
||||||
assert.Contains(t, err.Error(), "must contain a single token")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("TOKEN_FILE holding only comments behaves like an empty file", func(t *testing.T) {
|
|
||||||
tokenFile := filepath.Join(t.TempDir(), "token")
|
|
||||||
require.NoError(t, os.WriteFile(tokenFile, []byte("\n# only a comment\n"), 0o600))
|
|
||||||
|
|
||||||
t.Setenv("TOKEN_FILE", tokenFile)
|
|
||||||
|
|
||||||
token, err := getToken()
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.Equal(t, "", token)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("token from BESZEL_AGENT_TOKEN_FILE", func(t *testing.T) {
|
t.Run("token from BESZEL_AGENT_TOKEN_FILE", func(t *testing.T) {
|
||||||
// Create a temporary token file
|
// Create a temporary token file
|
||||||
expectedToken := "test-token-from-beszel-file"
|
expectedToken := "test-token-from-beszel-file"
|
||||||
@@ -722,11 +504,3 @@ func TestGetToken(t *testing.T) {
|
|||||||
assert.Equal(t, expectedToken, token, "Whitespace should be stripped from token file content")
|
assert.Equal(t, expectedToken, token, "Whitespace should be stripped from token file content")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebSocketDeadlineCoversSlowCollection(t *testing.T) {
|
|
||||||
const minimumDeadline = 120 * time.Second
|
|
||||||
|
|
||||||
if wsDeadline < minimumDeadline {
|
|
||||||
t.Fatalf("WebSocket deadline %s is shorter than the slow-collection window of %s", wsDeadline, minimumDeadline)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -87,19 +87,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
|
|||||||
|
|
||||||
wsClient, err := newWebSocketClient(c.agent)
|
wsClient, err := newWebSocketClient(c.agent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var caCertErr *caCertFileError
|
slog.Warn("Error creating WebSocket client", "err", err)
|
||||||
if errors.As(err, &caCertErr) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
disableSSH, _ := utils.GetEnv("DISABLE_SSH")
|
|
||||||
if errors.Is(err, errNoHubURL) && disableSSH != "true" {
|
|
||||||
// SSH-only mode: the hub dials the agent, so there is nothing to warn
|
|
||||||
// about. With SSH also disabled there is no connection method at all,
|
|
||||||
// so that case still warns.
|
|
||||||
slog.Debug("WebSocket client not configured", "err", err)
|
|
||||||
} else {
|
|
||||||
slog.Warn("Error creating WebSocket client", "err", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
c.wsClient = wsClient
|
c.wsClient = wsClient
|
||||||
|
|
||||||
@@ -153,7 +141,6 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
|
|||||||
// }
|
// }
|
||||||
func (c *ConnectionManager) stop() error {
|
func (c *ConnectionManager) stop() error {
|
||||||
_ = c.agent.StopServer()
|
_ = c.agent.StopServer()
|
||||||
c.agent.monitorManager.Stop()
|
|
||||||
c.closeWebSocket()
|
c.closeWebSocket()
|
||||||
return health.CleanUp()
|
return health.CleanUp()
|
||||||
}
|
}
|
||||||
@@ -164,9 +151,7 @@ func (c *ConnectionManager) handleEvent(event ConnectionEvent) {
|
|||||||
case WebSocketConnect:
|
case WebSocketConnect:
|
||||||
c.handleStateChange(WebSocketConnected)
|
c.handleStateChange(WebSocketConnected)
|
||||||
case SSHConnect:
|
case SSHConnect:
|
||||||
if c.State == Disconnected {
|
c.handleStateChange(SSHConnected)
|
||||||
c.handleStateChange(SSHConnected)
|
|
||||||
}
|
|
||||||
case WebSocketDisconnect:
|
case WebSocketDisconnect:
|
||||||
if c.State == WebSocketConnected {
|
if c.State == WebSocketConnected {
|
||||||
c.handleStateChange(Disconnected)
|
c.handleStateChange(Disconnected)
|
||||||
|
|||||||
@@ -114,12 +114,6 @@ func TestConnectionManager_EventHandling(t *testing.T) {
|
|||||||
event: SSHConnect,
|
event: SSHConnect,
|
||||||
expectedState: SSHConnected,
|
expectedState: SSHConnected,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "SSH connect from WebSocket connected (no change)",
|
|
||||||
initialState: WebSocketConnected,
|
|
||||||
event: SSHConnect,
|
|
||||||
expectedState: WebSocketConnected,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "WebSocket disconnect from connected",
|
name: "WebSocket disconnect from connected",
|
||||||
initialState: WebSocketConnected,
|
initialState: WebSocketConnected,
|
||||||
@@ -271,19 +265,6 @@ func TestConnectionManager_StartWithInvalidConfig(t *testing.T) {
|
|||||||
assert.Error(t, err, "Should error when starting already started connection manager")
|
assert.Error(t, err, "Should error when starting already started connection manager")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConnectionManager_StartRejectsInvalidCACertFile(t *testing.T) {
|
|
||||||
agent := createTestAgent(t)
|
|
||||||
cm := agent.connectionManager
|
|
||||||
t.Setenv("BESZEL_AGENT_HUB_URL", "https://hub.example.com")
|
|
||||||
t.Setenv("BESZEL_AGENT_TOKEN", "test-token")
|
|
||||||
t.Setenv("BESZEL_AGENT_CA_CERT_FILE", t.TempDir())
|
|
||||||
|
|
||||||
err := cm.Start(ServerOptions{})
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "read CA_CERT_FILE")
|
|
||||||
assert.Nil(t, cm.eventChan)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestConnectionManager_CloseWebSocket tests WebSocket closing
|
// TestConnectionManager_CloseWebSocket tests WebSocket closing
|
||||||
func TestConnectionManager_CloseWebSocket(t *testing.T) {
|
func TestConnectionManager_CloseWebSocket(t *testing.T) {
|
||||||
agent := createTestAgent(t)
|
agent := createTestAgent(t)
|
||||||
|
|||||||
+4
-12
@@ -12,14 +12,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func invalidDataDir(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
filePath := filepath.Join(t.TempDir(), "file")
|
|
||||||
require.NoError(t, os.WriteFile(filePath, nil, 0644))
|
|
||||||
return filepath.Join(filePath, "data")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDataDir(t *testing.T) {
|
func TestGetDataDir(t *testing.T) {
|
||||||
// Test with explicit dataDir parameter
|
// Test with explicit dataDir parameter
|
||||||
t.Run("explicit data dir", func(t *testing.T) {
|
t.Run("explicit data dir", func(t *testing.T) {
|
||||||
@@ -56,7 +48,7 @@ func TestGetDataDir(t *testing.T) {
|
|||||||
|
|
||||||
// Test with invalid explicit dataDir
|
// Test with invalid explicit dataDir
|
||||||
t.Run("invalid explicit data dir", func(t *testing.T) {
|
t.Run("invalid explicit data dir", func(t *testing.T) {
|
||||||
invalidPath := invalidDataDir(t)
|
invalidPath := "/invalid/path/that/cannot/be/created"
|
||||||
_, err := GetDataDir(invalidPath)
|
_, err := GetDataDir(invalidPath)
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
})
|
})
|
||||||
@@ -86,7 +78,7 @@ func TestTestDataDirs(t *testing.T) {
|
|||||||
// Test with multiple directories, first one valid
|
// Test with multiple directories, first one valid
|
||||||
t.Run("multiple dirs - first valid", func(t *testing.T) {
|
t.Run("multiple dirs - first valid", func(t *testing.T) {
|
||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
invalidDir := invalidDataDir(t)
|
invalidDir := "/invalid/path"
|
||||||
result, err := testDataDirs([]string{tempDir, invalidDir})
|
result, err := testDataDirs([]string{tempDir, invalidDir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, tempDir, result)
|
assert.Equal(t, tempDir, result)
|
||||||
@@ -95,7 +87,7 @@ func TestTestDataDirs(t *testing.T) {
|
|||||||
// Test with multiple directories, second one valid
|
// Test with multiple directories, second one valid
|
||||||
t.Run("multiple dirs - second valid", func(t *testing.T) {
|
t.Run("multiple dirs - second valid", func(t *testing.T) {
|
||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
invalidDir := invalidDataDir(t)
|
invalidDir := "/invalid/path"
|
||||||
result, err := testDataDirs([]string{invalidDir, tempDir})
|
result, err := testDataDirs([]string{invalidDir, tempDir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, tempDir, result)
|
assert.Equal(t, tempDir, result)
|
||||||
@@ -117,7 +109,7 @@ func TestTestDataDirs(t *testing.T) {
|
|||||||
|
|
||||||
// Test with no valid directories
|
// Test with no valid directories
|
||||||
t.Run("no valid directories", func(t *testing.T) {
|
t.Run("no valid directories", func(t *testing.T) {
|
||||||
invalidPaths := []string{invalidDataDir(t), invalidDataDir(t)}
|
invalidPaths := []string{"/invalid/path1", "/invalid/path2"}
|
||||||
_, err := testDataDirs(invalidPaths)
|
_, err := testDataDirs(invalidPaths)
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
assert.Contains(t, err.Error(), "data directory not found")
|
assert.Contains(t, err.Error(), "data directory not found")
|
||||||
|
|||||||
+11
-44
@@ -18,8 +18,7 @@ import (
|
|||||||
// fsRegistrationContext holds the shared lookup state needed to resolve a
|
// fsRegistrationContext holds the shared lookup state needed to resolve a
|
||||||
// filesystem into the tracked fsStats key and metadata.
|
// filesystem into the tracked fsStats key and metadata.
|
||||||
type fsRegistrationContext struct {
|
type fsRegistrationContext struct {
|
||||||
filesystem string // device part of optional FILESYSTEM env var
|
filesystem string // value of optional FILESYSTEM env var
|
||||||
filesystemName string // optional custom name from FILESYSTEM=device__name
|
|
||||||
isWindows bool
|
isWindows bool
|
||||||
efPath string // path to extra filesystems (default "/extra-filesystems")
|
efPath string // path to extra filesystems (default "/extra-filesystems")
|
||||||
diskIoCounters map[string]disk.IOCountersStat
|
diskIoCounters map[string]disk.IOCountersStat
|
||||||
@@ -178,7 +177,7 @@ func (d *diskDiscovery) addConfiguredRootFs() bool {
|
|||||||
|
|
||||||
for _, p := range d.partitions {
|
for _, p := range d.partitions {
|
||||||
if filesystemMatchesPartitionSetting(d.ctx.filesystem, p) {
|
if filesystemMatchesPartitionSetting(d.ctx.filesystem, p) {
|
||||||
d.addFsStat(p.Device, p.Mountpoint, true, d.ctx.filesystemName)
|
d.addFsStat(p.Device, p.Mountpoint, true, "")
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -186,7 +185,7 @@ func (d *diskDiscovery) addConfiguredRootFs() bool {
|
|||||||
// FILESYSTEM may name a physical disk absent from partitions (e.g. ZFS lists
|
// FILESYSTEM may name a physical disk absent from partitions (e.g. ZFS lists
|
||||||
// dataset paths like zroot/ROOT/default, not block devices).
|
// dataset paths like zroot/ROOT/default, not block devices).
|
||||||
if ioKey, match := findIoDevice(d.ctx.filesystem, d.ctx.diskIoCounters); match {
|
if ioKey, match := findIoDevice(d.ctx.filesystem, d.ctx.diskIoCounters); match {
|
||||||
d.agent.fsStats[ioKey] = &system.FsStats{Root: true, Mountpoint: d.rootMountPoint, Name: d.ctx.filesystemName}
|
d.agent.fsStats[ioKey] = &system.FsStats{Root: true, Mountpoint: d.rootMountPoint}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,8 +300,7 @@ func (d *diskDiscovery) addExtraFilesystemFolders(folderNames []string) {
|
|||||||
|
|
||||||
// Sets up the filesystems to monitor for disk usage and I/O.
|
// Sets up the filesystems to monitor for disk usage and I/O.
|
||||||
func (a *Agent) initializeDiskInfo() {
|
func (a *Agent) initializeDiskInfo() {
|
||||||
filesystemRaw, _ := utils.GetEnv("FILESYSTEM")
|
filesystem, _ := utils.GetEnv("FILESYSTEM")
|
||||||
filesystem, filesystemName := parseFilesystemEntry(filesystemRaw)
|
|
||||||
hasRoot := false
|
hasRoot := false
|
||||||
isWindows := runtime.GOOS == "windows"
|
isWindows := runtime.GOOS == "windows"
|
||||||
|
|
||||||
@@ -326,7 +324,6 @@ func (a *Agent) initializeDiskInfo() {
|
|||||||
slog.Debug("Disk I/O", "diskstats", diskIoCounters)
|
slog.Debug("Disk I/O", "diskstats", diskIoCounters)
|
||||||
ctx := fsRegistrationContext{
|
ctx := fsRegistrationContext{
|
||||||
filesystem: filesystem,
|
filesystem: filesystem,
|
||||||
filesystemName: filesystemName,
|
|
||||||
isWindows: isWindows,
|
isWindows: isWindows,
|
||||||
diskIoCounters: diskIoCounters,
|
diskIoCounters: diskIoCounters,
|
||||||
efPath: "/extra-filesystems",
|
efPath: "/extra-filesystems",
|
||||||
@@ -537,16 +534,7 @@ func normalizeDeviceName(value string) string {
|
|||||||
func (a *Agent) initializeDiskIoStats(diskIoCounters map[string]disk.IOCountersStat) {
|
func (a *Agent) initializeDiskIoStats(diskIoCounters map[string]disk.IOCountersStat) {
|
||||||
a.fsNames = a.fsNames[:0]
|
a.fsNames = a.fsNames[:0]
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
// ZFS datasets have no /proc/diskstats entry, so they are excluded from
|
|
||||||
// I/O tracking instead of warning about a missing device (#1541).
|
|
||||||
var zfsMountpoints map[string]bool
|
|
||||||
if a.storagePoolManager != nil {
|
|
||||||
zfsMountpoints = a.storagePoolManager.ZfsMountpoints()
|
|
||||||
}
|
|
||||||
for device, stats := range a.fsStats {
|
for device, stats := range a.fsStats {
|
||||||
if zfsMountpoints[stats.Mountpoint] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// skip if not in diskIoCounters
|
// skip if not in diskIoCounters
|
||||||
d, exists := diskIoCounters[device]
|
d, exists := diskIoCounters[device]
|
||||||
if !exists {
|
if !exists {
|
||||||
@@ -571,31 +559,20 @@ func (a *Agent) updateDiskUsage(systemStats *system.Stats) {
|
|||||||
!a.lastDiskUsageUpdate.IsZero() &&
|
!a.lastDiskUsageUpdate.IsZero() &&
|
||||||
time.Since(a.lastDiskUsageUpdate) < a.diskUsageCacheDuration
|
time.Since(a.lastDiskUsageUpdate) < a.diskUsageCacheDuration
|
||||||
|
|
||||||
// ZFS dataset mountpoints use `zfs list` values because statfs(2) reports
|
|
||||||
// dataset-level usage that excludes child datasets (#1541).
|
|
||||||
var zfsUsage map[string]zfsDatasetUsage
|
|
||||||
if a.storagePoolManager != nil {
|
|
||||||
zfsUsage = a.storagePoolManager.DatasetUsage()
|
|
||||||
}
|
|
||||||
|
|
||||||
// disk usage
|
// disk usage
|
||||||
for _, stats := range a.fsStats {
|
for _, stats := range a.fsStats {
|
||||||
// Skip non-root filesystems if caching is active
|
// Skip non-root filesystems if caching is active
|
||||||
if cacheExtraFs && !stats.Root {
|
if cacheExtraFs && !stats.Root {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var total, used uint64
|
if d, err := disk.Usage(stats.Mountpoint); err == nil {
|
||||||
var usedPct float64
|
stats.DiskTotal = utils.BytesToGigabytes(d.Total)
|
||||||
if u, ok := zfsUsage[stats.Mountpoint]; ok {
|
stats.DiskUsed = utils.BytesToGigabytes(d.Used)
|
||||||
total = u.used + u.avail
|
if stats.Root {
|
||||||
used = u.used
|
systemStats.DiskTotal = utils.BytesToGigabytes(d.Total)
|
||||||
if total > 0 {
|
systemStats.DiskUsed = utils.BytesToGigabytes(d.Used)
|
||||||
usedPct = float64(used) / float64(total) * 100
|
systemStats.DiskPct = utils.TwoDecimals(d.UsedPercent)
|
||||||
}
|
}
|
||||||
} else if d, err := disk.Usage(stats.Mountpoint); err == nil {
|
|
||||||
total = d.Total
|
|
||||||
used = d.Used
|
|
||||||
usedPct = d.UsedPercent
|
|
||||||
} else {
|
} else {
|
||||||
// reset stats if error (likely unmounted)
|
// reset stats if error (likely unmounted)
|
||||||
slog.Error("Error getting disk stats", "name", stats.Mountpoint, "err", err)
|
slog.Error("Error getting disk stats", "name", stats.Mountpoint, "err", err)
|
||||||
@@ -603,14 +580,6 @@ func (a *Agent) updateDiskUsage(systemStats *system.Stats) {
|
|||||||
stats.DiskUsed = 0
|
stats.DiskUsed = 0
|
||||||
stats.TotalRead = 0
|
stats.TotalRead = 0
|
||||||
stats.TotalWrite = 0
|
stats.TotalWrite = 0
|
||||||
continue
|
|
||||||
}
|
|
||||||
stats.DiskTotal = utils.BytesToGigabytes(total)
|
|
||||||
stats.DiskUsed = utils.BytesToGigabytes(used)
|
|
||||||
if stats.Root {
|
|
||||||
systemStats.DiskTotal = stats.DiskTotal
|
|
||||||
systemStats.DiskUsed = stats.DiskUsed
|
|
||||||
systemStats.DiskPct = utils.TwoDecimals(usedPct)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -727,8 +696,6 @@ func (a *Agent) updateDiskIo(cacheTimeMs uint16, systemStats *system.Stats) {
|
|||||||
systemStats.DiskWritePs = stats.DiskWritePs
|
systemStats.DiskWritePs = stats.DiskWritePs
|
||||||
systemStats.DiskIO[0] = diskIORead
|
systemStats.DiskIO[0] = diskIORead
|
||||||
systemStats.DiskIO[1] = diskIOWrite
|
systemStats.DiskIO[1] = diskIOWrite
|
||||||
systemStats.DiskIOTotal[0] = d.ReadBytes
|
|
||||||
systemStats.DiskIOTotal[1] = d.WriteBytes
|
|
||||||
systemStats.DiskIoStats[0] = diskReadTime
|
systemStats.DiskIoStats[0] = diskReadTime
|
||||||
systemStats.DiskIoStats[1] = diskWriteTime
|
systemStats.DiskIoStats[1] = diskWriteTime
|
||||||
systemStats.DiskIoStats[2] = diskIoUtilPct
|
systemStats.DiskIoStats[2] = diskIoUtilPct
|
||||||
|
|||||||
+12
-9
@@ -78,7 +78,14 @@ func TestParseFilesystemEntry(t *testing.T) {
|
|||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
fs, customName := parseFilesystemEntry(tt.input)
|
fsEntry := strings.TrimSpace(tt.input)
|
||||||
|
var fs, customName string
|
||||||
|
if parts := strings.SplitN(fsEntry, "__", 2); len(parts) == 2 {
|
||||||
|
fs = strings.TrimSpace(parts[0])
|
||||||
|
customName = strings.TrimSpace(parts[1])
|
||||||
|
} else {
|
||||||
|
fs = fsEntry
|
||||||
|
}
|
||||||
|
|
||||||
assert.Equal(t, tt.expectedFs, fs)
|
assert.Equal(t, tt.expectedFs, fs)
|
||||||
assert.Equal(t, tt.expectedName, customName)
|
assert.Equal(t, tt.expectedName, customName)
|
||||||
@@ -280,9 +287,8 @@ func TestAddConfiguredRootFs(t *testing.T) {
|
|||||||
rootMountPoint: "/",
|
rootMountPoint: "/",
|
||||||
partitions: []disk.PartitionStat{{Device: "/dev/ada0p2", Mountpoint: "/"}},
|
partitions: []disk.PartitionStat{{Device: "/dev/ada0p2", Mountpoint: "/"}},
|
||||||
ctx: fsRegistrationContext{
|
ctx: fsRegistrationContext{
|
||||||
filesystem: "/dev/ada0p2",
|
filesystem: "/dev/ada0p2",
|
||||||
filesystemName: "root disk",
|
isWindows: false,
|
||||||
isWindows: false,
|
|
||||||
diskIoCounters: map[string]disk.IOCountersStat{
|
diskIoCounters: map[string]disk.IOCountersStat{
|
||||||
"ada0": {Name: "ada0", ReadBytes: 1000, WriteBytes: 1000},
|
"ada0": {Name: "ada0", ReadBytes: 1000, WriteBytes: 1000},
|
||||||
},
|
},
|
||||||
@@ -296,7 +302,6 @@ func TestAddConfiguredRootFs(t *testing.T) {
|
|||||||
assert.True(t, exists)
|
assert.True(t, exists)
|
||||||
assert.True(t, stats.Root)
|
assert.True(t, stats.Root)
|
||||||
assert.Equal(t, "/", stats.Mountpoint)
|
assert.Equal(t, "/", stats.Mountpoint)
|
||||||
assert.Equal(t, "root disk", stats.Name)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("adds root from io device when partition is missing", func(t *testing.T) {
|
t.Run("adds root from io device when partition is missing", func(t *testing.T) {
|
||||||
@@ -305,9 +310,8 @@ func TestAddConfiguredRootFs(t *testing.T) {
|
|||||||
agent: agent,
|
agent: agent,
|
||||||
rootMountPoint: "/sysroot",
|
rootMountPoint: "/sysroot",
|
||||||
ctx: fsRegistrationContext{
|
ctx: fsRegistrationContext{
|
||||||
filesystem: "zroot",
|
filesystem: "zroot",
|
||||||
filesystemName: "root pool",
|
isWindows: false,
|
||||||
isWindows: false,
|
|
||||||
diskIoCounters: map[string]disk.IOCountersStat{
|
diskIoCounters: map[string]disk.IOCountersStat{
|
||||||
"nda0": {Name: "nda0", Label: "zroot", ReadBytes: 1000, WriteBytes: 1000},
|
"nda0": {Name: "nda0", Label: "zroot", ReadBytes: 1000, WriteBytes: 1000},
|
||||||
},
|
},
|
||||||
@@ -321,7 +325,6 @@ func TestAddConfiguredRootFs(t *testing.T) {
|
|||||||
assert.True(t, exists)
|
assert.True(t, exists)
|
||||||
assert.True(t, stats.Root)
|
assert.True(t, stats.Root)
|
||||||
assert.Equal(t, "/sysroot", stats.Mountpoint)
|
assert.Equal(t, "/sysroot", stats.Mountpoint)
|
||||||
assert.Equal(t, "root pool", stats.Name)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("returns false when filesystem cannot be resolved", func(t *testing.T) {
|
t.Run("returns false when filesystem cannot be resolved", func(t *testing.T) {
|
||||||
|
|||||||
@@ -1,110 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/shirou/gopsutil/v4/disk"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestUpdateDiskUsageZfsMountpoint verifies that a filesystem whose mountpoint
|
|
||||||
// is a ZFS dataset reports `zfs list` usage (which includes child datasets)
|
|
||||||
// instead of the dataset-scoped statfs values (#1541).
|
|
||||||
func TestUpdateDiskUsageZfsMountpoint(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
agent := &Agent{
|
|
||||||
fsStats: map[string]*system.FsStats{
|
|
||||||
"tank": {Root: false, Mountpoint: "/tank"},
|
|
||||||
},
|
|
||||||
storagePoolManager: zm,
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
agent.updateDiskUsage(&stats)
|
|
||||||
|
|
||||||
fs := agent.fsStats["tank"]
|
|
||||||
require.NotNil(t, fs)
|
|
||||||
assert.Equal(t, 22350.81, fs.DiskTotal) // (used + avail) in GiB
|
|
||||||
assert.Equal(t, 11175.87, fs.DiskUsed)
|
|
||||||
// Non-root filesystems do not populate system-level stats.
|
|
||||||
assert.Equal(t, float64(0), stats.DiskTotal)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdateDiskUsageZfsRootPopulatesSystemStats verifies the root disk values
|
|
||||||
// are derived from ZFS usage when the root mountpoint is a ZFS dataset.
|
|
||||||
func TestUpdateDiskUsageZfsRootPopulatesSystemStats(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "rpool/ROOT/pve-1", Used: 900000000000, Avail: 300000000000, Mountpoint: "/"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
agent := &Agent{
|
|
||||||
fsStats: map[string]*system.FsStats{
|
|
||||||
"rpool/ROOT/pve-1": {Root: true, Mountpoint: "/"},
|
|
||||||
},
|
|
||||||
storagePoolManager: zm,
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
agent.updateDiskUsage(&stats)
|
|
||||||
|
|
||||||
assert.Equal(t, 1117.59, agent.fsStats["rpool/ROOT/pve-1"].DiskTotal)
|
|
||||||
assert.Equal(t, 838.19, agent.fsStats["rpool/ROOT/pve-1"].DiskUsed)
|
|
||||||
assert.Equal(t, 75.0, stats.DiskPct)
|
|
||||||
assert.Equal(t, 1117.59, stats.DiskTotal)
|
|
||||||
assert.Equal(t, 838.19, stats.DiskUsed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdateDiskUsageWithoutZfsManager falls back to statfs when no manager is
|
|
||||||
// present (e.g. tests constructing bare Agent values).
|
|
||||||
func TestUpdateDiskUsageWithoutZfsManager(t *testing.T) {
|
|
||||||
agent := &Agent{
|
|
||||||
fsStats: map[string]*system.FsStats{
|
|
||||||
"root": {Root: true, Mountpoint: "/"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
agent.updateDiskUsage(&stats)
|
|
||||||
|
|
||||||
assert.True(t, agent.fsStats["root"].DiskTotal > 0, "root usage should come from statfs")
|
|
||||||
assert.True(t, stats.DiskTotal > 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestInitializeDiskIoStatsSkipsZfsMountpoints verifies ZFS filesystems are
|
|
||||||
// excluded from diskstats I/O tracking instead of warning about a missing device.
|
|
||||||
func TestInitializeDiskIoStatsSkipsZfsMountpoints(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{{Name: "tank", Mountpoint: "/tank"}}, nil
|
|
||||||
}
|
|
||||||
agent := &Agent{
|
|
||||||
fsStats: map[string]*system.FsStats{
|
|
||||||
"tank": {Root: false, Mountpoint: "/tank"},
|
|
||||||
"sda1": {Root: false, Mountpoint: "/mnt/data"},
|
|
||||||
},
|
|
||||||
storagePoolManager: zm,
|
|
||||||
diskPrev: make(map[uint16]map[string]prevDisk),
|
|
||||||
}
|
|
||||||
|
|
||||||
agent.initializeDiskIoStats(map[string]disk.IOCountersStat{
|
|
||||||
"sda1": {Name: "sda1", ReadBytes: 100, WriteBytes: 100},
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(t, []string{"sda1"}, agent.fsNames)
|
|
||||||
assert.Equal(t, uint64(100), agent.fsStats["sda1"].TotalRead)
|
|
||||||
// ZFS entry is present but untouched by diskstats initialization.
|
|
||||||
assert.Equal(t, uint64(0), agent.fsStats["tank"].TotalRead)
|
|
||||||
}
|
|
||||||
+9
-30
@@ -65,14 +65,10 @@ type dockerManager struct {
|
|||||||
dockerVersionChecked bool // Whether a version probe has completed successfully
|
dockerVersionChecked bool // Whether a version probe has completed successfully
|
||||||
isWindows bool // Whether the Docker Engine API is running on Windows
|
isWindows bool // Whether the Docker Engine API is running on Windows
|
||||||
buf *bytes.Buffer // Buffer to store and read response bodies
|
buf *bytes.Buffer // Buffer to store and read response bodies
|
||||||
|
apiStats *container.ApiStats // Reusable API stats object
|
||||||
excludeContainers []string // Patterns to exclude containers by name
|
excludeContainers []string // Patterns to exclude containers by name
|
||||||
usingPodman bool // Whether the Docker Engine API is running on Podman
|
usingPodman bool // Whether the Docker Engine API is running on Podman
|
||||||
|
|
||||||
registryClient *http.Client // Client for registry requests; nil uses a client with a 10-second timeout
|
|
||||||
imageUpdatesMutex sync.RWMutex // Protects imageUpdates, its entries, and imageUpdatesRunning
|
|
||||||
imageUpdates map[string]*imageUpdateStatus // Shared update status keyed by normalized image reference
|
|
||||||
imageUpdatesRunning bool // Whether a background image-update batch is in progress
|
|
||||||
|
|
||||||
// Cache-time-aware tracking for CPU stats (similar to cpu.go)
|
// Cache-time-aware tracking for CPU stats (similar to cpu.go)
|
||||||
// Maps cache time intervals to container-specific CPU usage tracking
|
// Maps cache time intervals to container-specific CPU usage tracking
|
||||||
lastCpuContainer map[uint16]map[string]uint64 // cacheTimeMs -> containerId -> last cpu container usage
|
lastCpuContainer map[uint16]map[string]uint64 // cacheTimeMs -> containerId -> last cpu container usage
|
||||||
@@ -165,9 +161,6 @@ func (dm *dockerManager) getDockerStats(cacheTimeMs uint16) ([]*container.Stats,
|
|||||||
clear(dm.validIds)
|
clear(dm.validIds)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only schedule auxiliary work here; metrics never wait for image discovery.
|
|
||||||
dm.refreshImageUpdates(dm.apiContainerList, time.Now())
|
|
||||||
|
|
||||||
var failedContainers []*container.ApiInfo
|
var failedContainers []*container.ApiInfo
|
||||||
|
|
||||||
for _, ctr := range dm.apiContainerList {
|
for _, ctr := range dm.apiContainerList {
|
||||||
@@ -513,17 +506,6 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read and decode the response before locking shared stats to avoid blocking
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return fmt.Errorf("container stats request failed: %s", resp.Status)
|
|
||||||
}
|
|
||||||
res := &container.ApiStats{}
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
updateAvailable := dm.cachedImageUpdate(ctr.Image)
|
|
||||||
|
|
||||||
dm.containerStatsMutex.Lock()
|
dm.containerStatsMutex.Lock()
|
||||||
defer dm.containerStatsMutex.Unlock()
|
defer dm.containerStatsMutex.Unlock()
|
||||||
|
|
||||||
@@ -538,9 +520,6 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
|
|||||||
stats.Status = statusText
|
stats.Status = statusText
|
||||||
stats.Health = health
|
stats.Health = health
|
||||||
|
|
||||||
stats.Image = ctr.Image
|
|
||||||
stats.UpdateAvailable = updateAvailable
|
|
||||||
|
|
||||||
if len(ctr.Ports) > 0 {
|
if len(ctr.Ports) > 0 {
|
||||||
stats.Ports = convertContainerPortsToString(ctr)
|
stats.Ports = convertContainerPortsToString(ctr)
|
||||||
}
|
}
|
||||||
@@ -553,24 +532,23 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
|
|||||||
stats.NetworkSent = 0
|
stats.NetworkSent = 0
|
||||||
stats.NetworkRecv = 0
|
stats.NetworkRecv = 0
|
||||||
|
|
||||||
|
res := dm.apiStats
|
||||||
|
res.Networks = nil
|
||||||
|
if err := dm.decode(resp, res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Initialize CPU tracking for this cache time interval
|
// Initialize CPU tracking for this cache time interval
|
||||||
dm.initializeCpuTracking(cacheTimeMs)
|
dm.initializeCpuTracking(cacheTimeMs)
|
||||||
|
|
||||||
// Get previous CPU values
|
// Get previous CPU values
|
||||||
prevCpuContainer, prevCpuSystem := dm.getCpuPreviousValues(cacheTimeMs, ctr.IdShort)
|
prevCpuContainer, prevCpuSystem := dm.getCpuPreviousValues(cacheTimeMs, ctr.IdShort)
|
||||||
|
|
||||||
// Calculate CPU percentage based on platform.
|
// Calculate CPU percentage based on platform
|
||||||
// Podman reports system_cpu_usage from cgroup cpu.stat (not /proc/stat), so it reflects
|
|
||||||
// only cgroup-tracked activity rather than total host capacity. Use a time-based method
|
|
||||||
// instead so the result is comparable to host CPU utilization. See:
|
|
||||||
// https://github.com/henrygd/beszel/issues/2049
|
|
||||||
var cpuPct float64
|
var cpuPct float64
|
||||||
if dm.isWindows {
|
if dm.isWindows {
|
||||||
prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]
|
prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]
|
||||||
cpuPct = res.CalculateCpuPercentWindows(prevCpuContainer, prevRead)
|
cpuPct = res.CalculateCpuPercentWindows(prevCpuContainer, prevRead)
|
||||||
} else if dm.usingPodman && res.CPUStats.OnlineCPUs > 0 {
|
|
||||||
prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]
|
|
||||||
cpuPct = res.CalculateCpuPercentPodman(prevCpuContainer, prevRead)
|
|
||||||
} else {
|
} else {
|
||||||
cpuPct = res.CalculateCpuPercentLinux(prevCpuContainer, prevCpuSystem)
|
cpuPct = res.CalculateCpuPercentLinux(prevCpuContainer, prevCpuSystem)
|
||||||
}
|
}
|
||||||
@@ -710,6 +688,7 @@ func newDockerManager(agent *Agent) *dockerManager {
|
|||||||
containerStatsMap: make(map[string]*container.Stats),
|
containerStatsMap: make(map[string]*container.Stats),
|
||||||
sem: make(chan struct{}, 5),
|
sem: make(chan struct{}, 5),
|
||||||
apiContainerList: []*container.ApiInfo{},
|
apiContainerList: []*container.ApiInfo{},
|
||||||
|
apiStats: &container.ApiStats{},
|
||||||
excludeContainers: excludeContainers,
|
excludeContainers: excludeContainers,
|
||||||
|
|
||||||
// Initialize cache-time-aware tracking structures
|
// Initialize cache-time-aware tracking structures
|
||||||
|
|||||||
@@ -1,105 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/distribution/reference"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
|
||||||
)
|
|
||||||
|
|
||||||
const imageUpdateInterval = time.Hour
|
|
||||||
|
|
||||||
type imageUpdateStatus struct {
|
|
||||||
available bool
|
|
||||||
checkedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func normalizedImageReference(image string) string {
|
|
||||||
named, err := reference.ParseNormalizedNamed(image)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
// Digest-pinned references cannot move to a new version.
|
|
||||||
if _, pinned := named.(reference.Digested); pinned {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return reference.TagNameOnly(named).String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// refreshImageUpdates starts at most one background batch. Neither its network
|
|
||||||
// work nor its completion is part of the container metrics wait group.
|
|
||||||
func (dm *dockerManager) refreshImageUpdates(containers []*container.ApiInfo, now time.Time) {
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
defer dm.imageUpdatesMutex.Unlock()
|
|
||||||
if dm.imageUpdatesRunning {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if dm.imageUpdates == nil {
|
|
||||||
dm.imageUpdates = make(map[string]*imageUpdateStatus)
|
|
||||||
}
|
|
||||||
active := make(map[string]struct{}, len(containers))
|
|
||||||
pending := make(map[string]*imageUpdateStatus)
|
|
||||||
for _, ctr := range containers {
|
|
||||||
if len(ctr.Names) > 0 && dm.shouldExcludeContainer(ctr.Names[0][1:]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := normalizedImageReference(ctr.Image)
|
|
||||||
if key == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
active[key] = struct{}{}
|
|
||||||
entry := dm.imageUpdates[key]
|
|
||||||
if entry == nil {
|
|
||||||
entry = &imageUpdateStatus{}
|
|
||||||
dm.imageUpdates[key] = entry
|
|
||||||
}
|
|
||||||
if entry.checkedAt.IsZero() || now.Sub(entry.checkedAt) >= imageUpdateInterval {
|
|
||||||
pending[key] = entry
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for key := range dm.imageUpdates {
|
|
||||||
if _, ok := active[key]; !ok {
|
|
||||||
delete(dm.imageUpdates, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(pending) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
dm.imageUpdatesRunning = true
|
|
||||||
go func() {
|
|
||||||
// Limit auxiliary requests even on hosts running many different images.
|
|
||||||
sem := make(chan struct{}, 2)
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for key, entry := range pending {
|
|
||||||
sem <- struct{}{}
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
defer func() { <-sem }()
|
|
||||||
available, err := dm.checkImageUpdate(key)
|
|
||||||
if err != nil {
|
|
||||||
available = false
|
|
||||||
slog.Debug("Image update check failed", "image", key, "err", err)
|
|
||||||
}
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
entry.available = available
|
|
||||||
entry.checkedAt = time.Now()
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
dm.imageUpdatesRunning = false
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *dockerManager) cachedImageUpdate(image string) bool {
|
|
||||||
key := normalizedImageReference(image)
|
|
||||||
dm.imageUpdatesMutex.RLock()
|
|
||||||
defer dm.imageUpdatesMutex.RUnlock()
|
|
||||||
entry := dm.imageUpdates[key]
|
|
||||||
return entry != nil && entry.available
|
|
||||||
}
|
|
||||||
@@ -1,225 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func waitForImageUpdates(t *testing.T, dm *dockerManager) {
|
|
||||||
t.Helper()
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
dm.imageUpdatesMutex.RLock()
|
|
||||||
defer dm.imageUpdatesMutex.RUnlock()
|
|
||||||
return !dm.imageUpdatesRunning
|
|
||||||
}, time.Second*3, time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageUpdateCacheAndStats(t *testing.T) {
|
|
||||||
local := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
remote := "sha256:" + strings.Repeat("b", 64)
|
|
||||||
var inspections, lookups atomic.Int32
|
|
||||||
var fail atomic.Bool
|
|
||||||
var upToDate atomic.Bool
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(r.URL.Path, "/images/"):
|
|
||||||
inspections.Add(1)
|
|
||||||
fmt.Fprintf(w, `{"RepoDigests":["docker.io/library/nginx@%s"]}`, local)
|
|
||||||
case r.URL.Path == "/containers/json":
|
|
||||||
fmt.Fprint(w, `[{"Id":"aaaaaaaaaaaa","Names":["/one"],"Image":"nginx","Status":"Up 2 hours"},{"Id":"bbbbbbbbbbbb","Names":["/two"],"Image":"docker.io/library/nginx:latest","Status":"Up 2 hours"}]`)
|
|
||||||
case strings.Contains(r.URL.Path, "/stats"):
|
|
||||||
fmt.Fprint(w, `{"memory_stats":{"usage":1048576},"cpu_stats":{},"networks":{}}`)
|
|
||||||
default:
|
|
||||||
http.NotFound(w, r)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
dm := newDockerManagerForVersionTest(server)
|
|
||||||
dm.dockerVersionChecked = true
|
|
||||||
dm.registryClient = &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
|
||||||
if fail.Load() {
|
|
||||||
return nil, fmt.Errorf("registry unavailable")
|
|
||||||
}
|
|
||||||
response := &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"token":"test"}`))}
|
|
||||||
if r.Method == http.MethodHead {
|
|
||||||
lookups.Add(1)
|
|
||||||
digest := remote
|
|
||||||
if upToDate.Load() {
|
|
||||||
digest = local
|
|
||||||
}
|
|
||||||
response.Header.Set("Docker-Content-Digest", digest)
|
|
||||||
}
|
|
||||||
return response, nil
|
|
||||||
})}
|
|
||||||
stats, err := dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, stats, 2)
|
|
||||||
waitForImageUpdates(t, dm)
|
|
||||||
require.EqualValues(t, 1, lookups.Load())
|
|
||||||
require.EqualValues(t, 1, inspections.Load())
|
|
||||||
stats, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, stat := range stats {
|
|
||||||
require.True(t, stat.UpdateAvailable)
|
|
||||||
if stat.Id == "aaaaaaaaaaaa" {
|
|
||||||
require.Equal(t, "nginx", stat.Image)
|
|
||||||
} else {
|
|
||||||
require.Equal(t, "docker.io/library/nginx:latest", stat.Image)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
require.EqualValues(t, 1, lookups.Load())
|
|
||||||
|
|
||||||
expire := func() {
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
dm.imageUpdates["docker.io/library/nginx:latest"].checkedAt = time.Now().Add(-imageUpdateInterval)
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
}
|
|
||||||
upToDate.Store(true)
|
|
||||||
expire()
|
|
||||||
_, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
waitForImageUpdates(t, dm)
|
|
||||||
require.EqualValues(t, 2, lookups.Load())
|
|
||||||
require.False(t, dm.cachedImageUpdate("nginx:latest"))
|
|
||||||
|
|
||||||
// An expired positive result is cleared on failure, and the failure itself
|
|
||||||
// is cached so realtime stats do not retry a broken registry every second.
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
dm.imageUpdates["docker.io/library/nginx:latest"].available = true
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
fail.Store(true)
|
|
||||||
expire()
|
|
||||||
_, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
waitForImageUpdates(t, dm)
|
|
||||||
failedInspections := inspections.Load()
|
|
||||||
stats, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, stats, 2)
|
|
||||||
require.Equal(t, failedInspections, inspections.Load())
|
|
||||||
for _, stat := range stats {
|
|
||||||
require.False(t, stat.UpdateAvailable)
|
|
||||||
require.Equal(t, 1.0, stat.Mem)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageDiscoveryDoesNotBlockStats(t *testing.T) {
|
|
||||||
started := make(chan struct{}, 1)
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/images/") {
|
|
||||||
fmt.Fprintf(w, `{"RepoDigests":["example.com/app@sha256:%s"]}`, strings.Repeat("a", 64))
|
|
||||||
} else {
|
|
||||||
fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
dm := newDockerManagerForVersionTest(server)
|
|
||||||
defer func() { close(release); waitForImageUpdates(t, dm) }()
|
|
||||||
dm.registryClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
|
||||||
started <- struct{}{}
|
|
||||||
<-release
|
|
||||||
return nil, fmt.Errorf("timeout")
|
|
||||||
})}
|
|
||||||
ctr := &container.ApiInfo{IdShort: "aaaaaaaaaaaa", Image: "example.com/app", Names: []string{"/one"}}
|
|
||||||
dm.refreshImageUpdates([]*container.ApiInfo{ctr}, time.Now())
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("check did not start")
|
|
||||||
}
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- dm.updateContainerStats(ctr, defaultCacheTimeMs) }()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.NoError(t, err)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("registry blocked stats")
|
|
||||||
}
|
|
||||||
dm.imageUpdatesMutex.RLock()
|
|
||||||
require.True(t, dm.imageUpdatesRunning)
|
|
||||||
dm.imageUpdatesMutex.RUnlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNormalizeImageUpdateReferences(t *testing.T) {
|
|
||||||
require.Equal(t, normalizedImageReference("nginx"), normalizedImageReference("docker.io/library/nginx:latest"))
|
|
||||||
require.Empty(t, normalizedImageReference("bad reference"))
|
|
||||||
require.Empty(t, normalizedImageReference("nginx@sha256:"+strings.Repeat("a", 64)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// A stats request can return headers promptly and then stall while reading its
|
|
||||||
// body. The stats-map mutex must remain available during that read.
|
|
||||||
func TestStatsResponseBodyDoesNotHoldStatsLock(t *testing.T) {
|
|
||||||
started := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
w.(http.Flusher).Flush()
|
|
||||||
close(started)
|
|
||||||
<-release
|
|
||||||
fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
dm := newDockerManagerForVersionTest(server)
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
done <- dm.updateContainerStats(&container.ApiInfo{IdShort: "aaaaaaaaaaaa", Names: []string{"/one"}, Image: "nginx"}, defaultCacheTimeMs)
|
|
||||||
}()
|
|
||||||
<-started
|
|
||||||
locked := make(chan struct{})
|
|
||||||
go func() { dm.containerStatsMutex.Lock(); dm.containerStatsMutex.Unlock(); close(locked) }()
|
|
||||||
select {
|
|
||||||
case <-locked:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
close(release)
|
|
||||||
<-done
|
|
||||||
t.Fatal("Docker response body held the stats mutex")
|
|
||||||
}
|
|
||||||
close(release)
|
|
||||||
require.NoError(t, <-done)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageUpdateStatsEncoding(t *testing.T) {
|
|
||||||
original := container.Stats{Image: "nginx:latest", UpdateAvailable: true}
|
|
||||||
encoded, err := cbor.Marshal(original)
|
|
||||||
require.NoError(t, err)
|
|
||||||
var fields map[int]any
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &fields))
|
|
||||||
require.Equal(t, true, fields[11])
|
|
||||||
require.Equal(t, "nginx:latest", fields[8])
|
|
||||||
var decoded container.Stats
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
|
|
||||||
require.True(t, decoded.UpdateAvailable)
|
|
||||||
require.Equal(t, original.Image, decoded.Image)
|
|
||||||
encoded, err = json.Marshal(original)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Contains(t, string(encoded), `"u":true`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageUpdateCacheExpiryBoundaryAndPruning(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
key := normalizedImageReference("nginx")
|
|
||||||
dm := &dockerManager{imageUpdates: map[string]*imageUpdateStatus{
|
|
||||||
key: {available: true, checkedAt: now},
|
|
||||||
"unused.example/image:latest": {checkedAt: now},
|
|
||||||
}}
|
|
||||||
dm.refreshImageUpdates([]*container.ApiInfo{{Image: "nginx"}}, now.Add(imageUpdateInterval-time.Nanosecond))
|
|
||||||
require.False(t, dm.imageUpdatesRunning)
|
|
||||||
require.Len(t, dm.imageUpdates, 1)
|
|
||||||
require.True(t, dm.cachedImageUpdate("nginx:latest"))
|
|
||||||
dm.refreshImageUpdates(nil, now)
|
|
||||||
require.Empty(t, dm.imageUpdates)
|
|
||||||
}
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
_ "crypto/sha256"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/distribution/reference"
|
|
||||||
"github.com/opencontainers/go-digest"
|
|
||||||
)
|
|
||||||
|
|
||||||
const imageRegistryTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
const imageManifestAccept = "application/vnd.docker.distribution.manifest.list.v2+json, " +
|
|
||||||
"application/vnd.docker.distribution.manifest.v2+json, " +
|
|
||||||
"application/vnd.oci.image.manifest.v1+json, " +
|
|
||||||
"application/vnd.oci.image.index.v1+json"
|
|
||||||
|
|
||||||
// checkImageUpdate compares the digest recorded by Docker for image with the
|
|
||||||
// digest currently advertised by its registry. A digest-pinned reference is
|
|
||||||
// immutable and therefore never has an update available.
|
|
||||||
func (dm *dockerManager) checkImageUpdate(image string) (bool, error) {
|
|
||||||
named, err := reference.ParseNormalizedNamed(image)
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("parse image reference %q: %w", image, err)
|
|
||||||
}
|
|
||||||
if _, pinned := named.(reference.Digested); pinned {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
named = reference.TagNameOnly(named)
|
|
||||||
|
|
||||||
registry := reference.Domain(named)
|
|
||||||
repository := reference.Path(named)
|
|
||||||
tag := named.(reference.Tagged).Tag()
|
|
||||||
|
|
||||||
localDigest, err := dm.inspectImageDigest(image, registry, repository)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
remoteDigest, err := dm.registryImageDigest(registry, repository, tag)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return remoteDigest != localDigest, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// inspectImageDigest reads Docker's image metadata without using dm.decode.
|
|
||||||
// The checker runs in the image-discovery goroutine, so it must not hold any
|
|
||||||
// of the container statistics locks while waiting on the Docker API.
|
|
||||||
func (dm *dockerManager) inspectImageDigest(image, registry, repository string) (string, error) {
|
|
||||||
if dm.client == nil {
|
|
||||||
return "", fmt.Errorf("inspect image %q: Docker client is unavailable", image)
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint := "http://localhost/images/" + url.PathEscape(image) + "/json"
|
|
||||||
resp, err := dm.client.Get(endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("inspect image %q: %w", image, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return "", fmt.Errorf("inspect image %q failed: %s", image, responseStatus(resp))
|
|
||||||
}
|
|
||||||
|
|
||||||
var inspect struct {
|
|
||||||
RepoDigests []string `json:"RepoDigests"`
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&inspect); err != nil {
|
|
||||||
return "", fmt.Errorf("decode image inspect %q: %w", image, err)
|
|
||||||
}
|
|
||||||
if len(inspect.RepoDigests) == 0 {
|
|
||||||
return "", fmt.Errorf("inspect image %q returned no repository digests", image)
|
|
||||||
}
|
|
||||||
|
|
||||||
localDigest, ok := matchingRepositoryDigest(inspect.RepoDigests, registry, repository)
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("inspect image %q returned no valid digest for %s/%s", image, registry, repository)
|
|
||||||
}
|
|
||||||
return localDigest, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// matchingRepositoryDigest returns a valid digest belonging to the requested
|
|
||||||
// repository. Docker can return multiple RepoDigests for one local image; an
|
|
||||||
// unrelated first entry must never be used for the comparison.
|
|
||||||
func matchingRepositoryDigest(repoDigests []string, registry, repository string) (string, bool) {
|
|
||||||
for _, repoDigest := range repoDigests {
|
|
||||||
repoDigest = strings.TrimSpace(repoDigest)
|
|
||||||
at := strings.LastIndexByte(repoDigest, '@')
|
|
||||||
if at <= 0 || at == len(repoDigest)-1 || strings.Contains(repoDigest[:at], "@") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
repoRef, err := reference.ParseNormalizedNamed(repoDigest[:at])
|
|
||||||
if err != nil || reference.Path(repoRef) != repository || !sameRegistry(reference.Domain(repoRef), registry) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, hasTag := repoRef.(reference.Tagged); hasTag {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
d, err := digest.Parse(repoDigest[at+1:])
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return d.String(), true
|
|
||||||
}
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
func sameRegistry(left, right string) bool {
|
|
||||||
left = canonicalRegistry(left)
|
|
||||||
right = canonicalRegistry(right)
|
|
||||||
return left == right ||
|
|
||||||
(left == "ghcr.io" && right == "lscr.io") ||
|
|
||||||
(left == "lscr.io" && right == "ghcr.io")
|
|
||||||
}
|
|
||||||
|
|
||||||
func canonicalRegistry(registry string) string {
|
|
||||||
if registry == "index.docker.io" {
|
|
||||||
return "docker.io"
|
|
||||||
}
|
|
||||||
return registry
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *dockerManager) registryImageDigest(registry, repository, tag string) (string, error) {
|
|
||||||
client := dm.registryClient
|
|
||||||
if client == nil {
|
|
||||||
client = &http.Client{Timeout: imageRegistryTimeout}
|
|
||||||
}
|
|
||||||
|
|
||||||
token, err := dm.registryToken(client, registry, repository)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
host := registry
|
|
||||||
if registry == "docker.io" {
|
|
||||||
host = "registry-1.docker.io"
|
|
||||||
}
|
|
||||||
manifestURL := "https://" + host + "/v2/" + repository + "/manifests/" + url.PathEscape(tag)
|
|
||||||
req, err := http.NewRequest(http.MethodHead, manifestURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("create manifest request: %w", err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Accept", imageManifestAccept)
|
|
||||||
if token != "" {
|
|
||||||
req.Header.Set("Authorization", "Bearer "+token)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("fetch manifest %s:%s: %w", registry, repository, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return "", fmt.Errorf("manifest request for %s:%s failed: %s", repository, tag, responseStatus(resp))
|
|
||||||
}
|
|
||||||
|
|
||||||
remote := strings.TrimSpace(resp.Header.Get("Docker-Content-Digest"))
|
|
||||||
d, err := digest.Parse(remote)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("manifest request for %s:%s returned invalid digest: %w", repository, tag, err)
|
|
||||||
}
|
|
||||||
return d.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *dockerManager) registryToken(client *http.Client, registry, repository string) (string, error) {
|
|
||||||
var authURL string
|
|
||||||
switch registry {
|
|
||||||
case "docker.io":
|
|
||||||
authURL = "https://auth.docker.io/token?service=registry.docker.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
|
|
||||||
case "ghcr.io", "lscr.io":
|
|
||||||
// lscr.io is the LinuxServer alias for its GHCR-backed images.
|
|
||||||
authURL = "https://ghcr.io/token?service=ghcr.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
|
|
||||||
default:
|
|
||||||
// Anonymous registries remain supported, as they were before the
|
|
||||||
// authenticated Docker Hub and GHCR paths were added.
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, authURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("create registry auth request: %w", err)
|
|
||||||
}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("fetch registry auth token for %s: %w", repository, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return "", fmt.Errorf("registry auth request for %s failed: %s", repository, responseStatus(resp))
|
|
||||||
}
|
|
||||||
|
|
||||||
var tokenResponse struct {
|
|
||||||
Token string `json:"token"`
|
|
||||||
AccessToken string `json:"access_token"`
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&tokenResponse); err != nil {
|
|
||||||
return "", fmt.Errorf("decode registry auth response for %s: %w", repository, err)
|
|
||||||
}
|
|
||||||
token := strings.TrimSpace(tokenResponse.Token)
|
|
||||||
if token == "" {
|
|
||||||
token = strings.TrimSpace(tokenResponse.AccessToken)
|
|
||||||
}
|
|
||||||
if token == "" {
|
|
||||||
return "", fmt.Errorf("registry auth response for %s contained no token", repository)
|
|
||||||
}
|
|
||||||
return token, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func responseStatus(resp *http.Response) string {
|
|
||||||
if resp.Status != "" {
|
|
||||||
return resp.Status
|
|
||||||
}
|
|
||||||
return http.StatusText(resp.StatusCode)
|
|
||||||
}
|
|
||||||
@@ -1,204 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
type registryTransportFunc func(*http.Request) (*http.Response, error)
|
|
||||||
|
|
||||||
func (fn registryTransportFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
|
||||||
return fn(req)
|
|
||||||
}
|
|
||||||
|
|
||||||
func registryResponse(status int, body string) *http.Response {
|
|
||||||
return &http.Response{
|
|
||||||
StatusCode: status,
|
|
||||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
|
||||||
Header: make(http.Header),
|
|
||||||
Body: io.NopCloser(strings.NewReader(body)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func registryDigest(fill byte) string {
|
|
||||||
return "sha256:" + strings.Repeat(string(fill), 64)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRegistryChecker(t *testing.T, inspectBody string, transport http.RoundTripper) *dockerManager {
|
|
||||||
t.Helper()
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/images/") {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_, _ = io.WriteString(w, inspectBody)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
http.NotFound(w, r)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
return &dockerManager{
|
|
||||||
client: newDockerManagerForVersionTest(server).client,
|
|
||||||
registryClient: &http.Client{Transport: transport},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateUsesInspectAndManifestDigests(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
remote := registryDigest('b')
|
|
||||||
var authCalls, manifestCalls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
switch {
|
|
||||||
case req.Method == http.MethodGet && req.URL.Host == "auth.docker.io":
|
|
||||||
authCalls.Add(1)
|
|
||||||
require.Equal(t, "/token", req.URL.Path)
|
|
||||||
return registryResponse(http.StatusOK, `{"token":"test-token"}`), nil
|
|
||||||
case req.Method == http.MethodHead && req.URL.Host == "registry-1.docker.io":
|
|
||||||
manifestCalls.Add(1)
|
|
||||||
require.Equal(t, "/v2/library/alpine/manifests/latest", req.URL.Path)
|
|
||||||
require.Equal(t, "Bearer test-token", req.Header.Get("Authorization"))
|
|
||||||
resp := registryResponse(http.StatusOK, "")
|
|
||||||
resp.Header.Set("Docker-Content-Digest", remote)
|
|
||||||
return resp, nil
|
|
||||||
default:
|
|
||||||
return registryResponse(http.StatusNotFound, ""), nil
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
|
|
||||||
available, err := dm.checkImageUpdate("alpine")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.True(t, available)
|
|
||||||
require.EqualValues(t, 1, authCalls.Load())
|
|
||||||
require.EqualValues(t, 1, manifestCalls.Load())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateReportsUnknownInspectState(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
body string
|
|
||||||
}{
|
|
||||||
{name: "missing field", body: `{}`},
|
|
||||||
{name: "empty field", body: `{"RepoDigests":[]}`},
|
|
||||||
{name: "malformed reference", body: `{"RepoDigests":["not-a-repo-digest"]}`},
|
|
||||||
{name: "wrong repository", body: `{"RepoDigests":["docker.io/library/busybox@` + registryDigest('a') + `"]}`},
|
|
||||||
{name: "malformed digest", body: `{"RepoDigests":["docker.io/library/alpine@sha256:not-a-digest"]}`},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
var registryCalls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, test.body, registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
registryCalls.Add(1)
|
|
||||||
return registryResponse(http.StatusOK, `{"token":"unexpected"}`), nil
|
|
||||||
}))
|
|
||||||
|
|
||||||
available, err := dm.checkImageUpdate("alpine")
|
|
||||||
require.Error(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
require.EqualValues(t, 0, registryCalls.Load(), "invalid local state must not query a registry")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateChecksInspectAuthAndManifestStatuses(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
validInspect := fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
inspectCode int
|
|
||||||
authCode int
|
|
||||||
manifestCode int
|
|
||||||
remote string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "inspect status", inspectCode: http.StatusNotFound, want: "inspect image"},
|
|
||||||
{name: "auth status", inspectCode: http.StatusOK, authCode: http.StatusUnauthorized, want: "registry auth"},
|
|
||||||
{name: "manifest status", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusNotFound, remote: local, want: "manifest request"},
|
|
||||||
{name: "missing digest", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusOK, want: "invalid digest"},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if test.inspectCode != http.StatusOK && strings.HasPrefix(r.URL.Path, "/images/") {
|
|
||||||
w.WriteHeader(test.inspectCode)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_, _ = io.WriteString(w, validInspect)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
calls := 0
|
|
||||||
dm := &dockerManager{client: newDockerManagerForVersionTest(server).client, registryClient: &http.Client{Transport: registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
calls++
|
|
||||||
if req.Method == http.MethodGet {
|
|
||||||
return registryResponse(test.authCode, `{"token":"test"}`), nil
|
|
||||||
}
|
|
||||||
response := registryResponse(test.manifestCode, "")
|
|
||||||
response.Header.Set("Docker-Content-Digest", test.remote)
|
|
||||||
return response, nil
|
|
||||||
})}}
|
|
||||||
|
|
||||||
_, err := dm.checkImageUpdate("alpine")
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, err.Error(), test.want)
|
|
||||||
if test.inspectCode != http.StatusOK {
|
|
||||||
require.Zero(t, calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateSupportsAnonymousAndLSCRRegistries(t *testing.T) {
|
|
||||||
t.Run("anonymous registry", func(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
var calls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["example.com/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
calls.Add(1)
|
|
||||||
require.Equal(t, http.MethodHead, req.Method)
|
|
||||||
require.Equal(t, "example.com", req.URL.Host)
|
|
||||||
resp := registryResponse(http.StatusOK, "")
|
|
||||||
resp.Header.Set("Docker-Content-Digest", local)
|
|
||||||
return resp, nil
|
|
||||||
}))
|
|
||||||
available, err := dm.checkImageUpdate("example.com/app")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
require.EqualValues(t, 1, calls.Load())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("lscr ghcr alias", func(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
var authCalls, manifestCalls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["ghcr.io/linuxserver/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
if req.Method == http.MethodGet {
|
|
||||||
authCalls.Add(1)
|
|
||||||
return registryResponse(http.StatusOK, `{"token":"test"}`), nil
|
|
||||||
}
|
|
||||||
manifestCalls.Add(1)
|
|
||||||
require.Equal(t, "lscr.io", req.URL.Host)
|
|
||||||
resp := registryResponse(http.StatusOK, "")
|
|
||||||
resp.Header.Set("Docker-Content-Digest", local)
|
|
||||||
return resp, nil
|
|
||||||
}))
|
|
||||||
available, err := dm.checkImageUpdate("lscr.io/linuxserver/app")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
require.EqualValues(t, 1, authCalls.Load())
|
|
||||||
require.EqualValues(t, 1, manifestCalls.Load())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateSkipsPinnedDigest(t *testing.T) {
|
|
||||||
image := "docker.io/library/alpine@" + registryDigest('a')
|
|
||||||
dm := &dockerManager{}
|
|
||||||
available, err := dm.checkImageUpdate(image)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
}
|
|
||||||
+1
-157
@@ -729,7 +729,6 @@ func TestGetDockerStatsChecksDockerVersionAfterContainerList(t *testing.T) {
|
|||||||
|
|
||||||
stats, err := dm.getDockerStats(defaultCacheTimeMs)
|
stats, err := dm.getDockerStats(defaultCacheTimeMs)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotNil(t, stats, "A successful empty snapshot must remain distinguishable from a collection failure")
|
|
||||||
assert.Empty(t, stats)
|
assert.Empty(t, stats)
|
||||||
assert.True(t, dm.dockerVersionChecked)
|
assert.True(t, dm.dockerVersionChecked)
|
||||||
assert.Equal(t, tt.expectedGood, dm.goodDockerVersion)
|
assert.Equal(t, tt.expectedGood, dm.goodDockerVersion)
|
||||||
@@ -743,7 +742,6 @@ func TestGetDockerStatsChecksDockerVersionAfterContainerList(t *testing.T) {
|
|||||||
|
|
||||||
stats, err = dm.getDockerStats(defaultCacheTimeMs)
|
stats, err = dm.getDockerStats(defaultCacheTimeMs)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotNil(t, stats, "A successful empty snapshot must remain distinguishable from a collection failure")
|
|
||||||
assert.Empty(t, stats)
|
assert.Empty(t, stats)
|
||||||
assert.Equal(t, tt.expectedGood, dm.goodDockerVersion)
|
assert.Equal(t, tt.expectedGood, dm.goodDockerVersion)
|
||||||
assert.Equal(t, tt.expectedPodman, dm.usingPodman)
|
assert.Equal(t, tt.expectedPodman, dm.usingPodman)
|
||||||
@@ -1061,161 +1059,6 @@ func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) {
|
|||||||
assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0)
|
assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCalculateCpuPercentPodman(t *testing.T) {
|
|
||||||
baseTime := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
prevCpuContainer uint64
|
|
||||||
prevRead time.Time
|
|
||||||
currentUsage uint64
|
|
||||||
currentRead time.Time
|
|
||||||
onlineCPUs uint32
|
|
||||||
expectedPct float64
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "normal calculation",
|
|
||||||
// container used 2ms of CPU over 1s with 2 CPUs → 0.1%
|
|
||||||
prevCpuContainer: 1_000_000_000,
|
|
||||||
prevRead: baseTime,
|
|
||||||
currentUsage: 1_002_000_000, // +2ms CPU time
|
|
||||||
currentRead: baseTime.Add(time.Second),
|
|
||||||
onlineCPUs: 2,
|
|
||||||
expectedPct: 0.1, // 2e6 / (1e9 * 2) * 100
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "first run returns zero",
|
|
||||||
prevCpuContainer: 0,
|
|
||||||
prevRead: baseTime,
|
|
||||||
currentUsage: 5_000_000,
|
|
||||||
currentRead: baseTime.Add(time.Second),
|
|
||||||
onlineCPUs: 4,
|
|
||||||
expectedPct: 0.0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "zero online cpus returns zero",
|
|
||||||
prevCpuContainer: 1_000_000_000,
|
|
||||||
prevRead: baseTime,
|
|
||||||
currentUsage: 1_010_000_000,
|
|
||||||
currentRead: baseTime.Add(time.Second),
|
|
||||||
onlineCPUs: 0,
|
|
||||||
expectedPct: 0.0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "same read time returns zero",
|
|
||||||
prevCpuContainer: 1_000_000_000,
|
|
||||||
prevRead: baseTime,
|
|
||||||
currentUsage: 1_010_000_000,
|
|
||||||
currentRead: baseTime, // no elapsed time
|
|
||||||
onlineCPUs: 2,
|
|
||||||
expectedPct: 0.0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "counter rollback returns zero",
|
|
||||||
prevCpuContainer: 2_000_000_000,
|
|
||||||
prevRead: baseTime,
|
|
||||||
currentUsage: 1_000_000_000,
|
|
||||||
currentRead: baseTime.Add(time.Second),
|
|
||||||
onlineCPUs: 2,
|
|
||||||
expectedPct: 0.0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "100% single cpu",
|
|
||||||
// container consumed a full CPU-second over 1s on a 1-CPU host → 100%
|
|
||||||
prevCpuContainer: 1_000_000_000,
|
|
||||||
prevRead: baseTime,
|
|
||||||
currentUsage: 2_000_000_000, // +1s CPU time
|
|
||||||
currentRead: baseTime.Add(time.Second),
|
|
||||||
onlineCPUs: 1,
|
|
||||||
expectedPct: 100.0, // 1e9 / (1e9 * 1) * 100
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "high utilization on multi-cpu host",
|
|
||||||
// container used 800ms on a 4-CPU host over 1s → 20%
|
|
||||||
prevCpuContainer: 10_000_000_000,
|
|
||||||
prevRead: baseTime,
|
|
||||||
currentUsage: 10_800_000_000,
|
|
||||||
currentRead: baseTime.Add(time.Second),
|
|
||||||
onlineCPUs: 4,
|
|
||||||
expectedPct: 20.0, // 800e6 / (1e9 * 4) * 100
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
s := &container.ApiStats{
|
|
||||||
Read: tt.currentRead,
|
|
||||||
CPUStats: container.CPUStats{
|
|
||||||
CPUUsage: container.CPUUsage{TotalUsage: tt.currentUsage},
|
|
||||||
OnlineCPUs: tt.onlineCPUs,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
got := s.CalculateCpuPercentPodman(tt.prevCpuContainer, tt.prevRead)
|
|
||||||
assert.InDelta(t, tt.expectedPct, got, 0.001, "test %q", tt.name)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {
|
|
||||||
// Verify that Podman containers use the time-based CPU calculation
|
|
||||||
// when online_cpus is provided in the stats response.
|
|
||||||
// container used 20ms CPU over 1s with 2 CPUs → 1%
|
|
||||||
prevReadTime := time.Date(2026, 3, 15, 21, 26, 58, 0, time.UTC) // 1 second before stats read
|
|
||||||
const prevCpuUsage = uint64(5_000_000_000)
|
|
||||||
|
|
||||||
dm := &dockerManager{
|
|
||||||
client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
switch req.URL.EscapedPath() {
|
|
||||||
case "/containers/0123456789ab/stats":
|
|
||||||
return &http.Response{
|
|
||||||
StatusCode: http.StatusOK,
|
|
||||||
Status: "200 OK",
|
|
||||||
Header: make(http.Header),
|
|
||||||
Body: io.NopCloser(strings.NewReader(`{
|
|
||||||
"read":"2026-03-15T21:26:59Z",
|
|
||||||
"cpu_stats":{"cpu_usage":{"total_usage":5020000000},"system_cpu_usage":9999999,"online_cpus":2},
|
|
||||||
"memory_stats":{"usage":1048576,"stats":{"inactive_file":262144}},
|
|
||||||
"networks":{"eth0":{"rx_bytes":0,"tx_bytes":0}}
|
|
||||||
}`)),
|
|
||||||
Request: req,
|
|
||||||
}, nil
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unexpected path: %s", req.URL.EscapedPath())
|
|
||||||
}
|
|
||||||
})},
|
|
||||||
containerStatsMap: make(map[string]*container.Stats),
|
|
||||||
usingPodman: true,
|
|
||||||
lastCpuContainer: map[uint16]map[string]uint64{
|
|
||||||
defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
|
|
||||||
},
|
|
||||||
lastCpuSystem: map[uint16]map[string]uint64{
|
|
||||||
defaultCacheTimeMs: {"0123456789ab": 1}, // intentionally tiny — should NOT be used
|
|
||||||
},
|
|
||||||
lastCpuReadTime: map[uint16]map[string]time.Time{
|
|
||||||
defaultCacheTimeMs: {"0123456789ab": prevReadTime},
|
|
||||||
},
|
|
||||||
networkSentTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
|
|
||||||
networkRecvTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
|
|
||||||
lastNetworkReadTime: make(map[uint16]map[string]time.Time),
|
|
||||||
}
|
|
||||||
|
|
||||||
ctr := &container.ApiInfo{
|
|
||||||
IdShort: "0123456789ab",
|
|
||||||
Names: []string{"/myapp"},
|
|
||||||
Status: "Up 5 minutes",
|
|
||||||
Image: "myapp:latest",
|
|
||||||
}
|
|
||||||
|
|
||||||
err := dm.updateContainerStats(ctr, defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// cpu delta = 5020000000 - 5000000000 = 20000000 ns (20ms)
|
|
||||||
// elapsed = 1s = 1000000000 ns, online_cpus = 2
|
|
||||||
// expected = 20000000 / (1000000000 * 2) * 100 = 1.0%
|
|
||||||
expectedCpu := 1.0
|
|
||||||
assert.InDelta(t, expectedCpu, dm.containerStatsMap[ctr.IdShort].Cpu, 0.01)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkStatsCalculationWithRealData(t *testing.T) {
|
func TestNetworkStatsCalculationWithRealData(t *testing.T) {
|
||||||
// Create synthetic test data to avoid timing issues
|
// Create synthetic test data to avoid timing issues
|
||||||
apiStats1 := &container.ApiStats{
|
apiStats1 := &container.ApiStats{
|
||||||
@@ -1675,6 +1518,7 @@ func TestUpdateContainerStatsUsesPodmanInspectHealthFallback(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})},
|
})},
|
||||||
containerStatsMap: make(map[string]*container.Stats),
|
containerStatsMap: make(map[string]*container.Stats),
|
||||||
|
apiStats: &container.ApiStats{},
|
||||||
usingPodman: true,
|
usingPodman: true,
|
||||||
lastCpuContainer: make(map[uint16]map[string]uint64),
|
lastCpuContainer: make(map[uint16]map[string]uint64),
|
||||||
lastCpuSystem: make(map[uint16]map[string]uint64),
|
lastCpuSystem: make(map[uint16]map[string]uint64),
|
||||||
|
|||||||
+3
-19
@@ -72,30 +72,14 @@ func discoverHwmonFans(root string) ([]fanSensor, error) {
|
|||||||
var sensors []fanSensor
|
var sensors []fanSensor
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
chipDir := filepath.Join(root, entry.Name())
|
chipDir := filepath.Join(root, entry.Name())
|
||||||
sensorDir := chipDir
|
chipName := utils.ReadStringFile(filepath.Join(chipDir, "name"))
|
||||||
inputs, _ := filepath.Glob(filepath.Join(sensorDir, "fan*_input"))
|
|
||||||
|
|
||||||
// Some legacy hwmon drivers (notably applesmc) register a hwmon class
|
|
||||||
// device but create fan attributes on the parent platform device. In
|
|
||||||
// sysfs that parent is exposed through hwmonN/device.
|
|
||||||
if len(inputs) == 0 {
|
|
||||||
deviceDir := filepath.Join(chipDir, "device")
|
|
||||||
if deviceInputs, _ := filepath.Glob(filepath.Join(deviceDir, "fan*_input")); len(deviceInputs) > 0 {
|
|
||||||
sensorDir = deviceDir
|
|
||||||
inputs = deviceInputs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
chipName := utils.ReadStringFile(filepath.Join(sensorDir, "name"))
|
|
||||||
if chipName == "" {
|
|
||||||
chipName = utils.ReadStringFile(filepath.Join(chipDir, "name"))
|
|
||||||
}
|
|
||||||
if chipName == "" {
|
if chipName == "" {
|
||||||
chipName = entry.Name()
|
chipName = entry.Name()
|
||||||
}
|
}
|
||||||
|
inputs, _ := filepath.Glob(filepath.Join(chipDir, "fan*_input"))
|
||||||
for _, inputPath := range inputs {
|
for _, inputPath := range inputs {
|
||||||
base := strings.TrimSuffix(filepath.Base(inputPath), "_input")
|
base := strings.TrimSuffix(filepath.Base(inputPath), "_input")
|
||||||
label := utils.ReadStringFile(filepath.Join(sensorDir, base+"_label"))
|
label := utils.ReadStringFile(filepath.Join(chipDir, base+"_label"))
|
||||||
key := chipName + "_" + base
|
key := chipName + "_" + base
|
||||||
if label != "" {
|
if label != "" {
|
||||||
key = chipName + "_" + label
|
key = chipName + "_" + label
|
||||||
|
|||||||
@@ -50,24 +50,6 @@ func TestReadHwmonFans(t *testing.T) {
|
|||||||
}, fans)
|
}, fans)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestReadHwmonFansLegacyParent verifies legacy hwmon layouts such as applesmc,
|
|
||||||
// where the hwmon class node exists but fan attributes live on hwmonN/device.
|
|
||||||
func TestReadHwmonFansLegacyParent(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
deviceDir := filepath.Join(root, "devices", "applesmc.768")
|
|
||||||
writeFile(t, filepath.Join(deviceDir, "name"), "applesmc\n")
|
|
||||||
writeFile(t, filepath.Join(deviceDir, "fan1_input"), "1202\n")
|
|
||||||
writeFile(t, filepath.Join(deviceDir, "fan1_label"), "Exhaust\n")
|
|
||||||
|
|
||||||
chipDir := filepath.Join(root, "hwmon1")
|
|
||||||
require.NoError(t, os.MkdirAll(chipDir, 0o755))
|
|
||||||
require.NoError(t, os.Symlink(deviceDir, filepath.Join(chipDir, "device")))
|
|
||||||
|
|
||||||
fans, err := readHwmonFans(root)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, map[string]uint16{"applesmc_Exhaust": 1202}, fans)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReadHwmonFansMissingRoot returns an error rather than panicking when the
|
// TestReadHwmonFansMissingRoot returns an error rather than panicking when the
|
||||||
// hwmon root doesn't exist (e.g. running on a kernel without hwmon support).
|
// hwmon root doesn't exist (e.g. running on a kernel without hwmon support).
|
||||||
func TestReadHwmonFansMissingRoot(t *testing.T) {
|
func TestReadHwmonFansMissingRoot(t *testing.T) {
|
||||||
|
|||||||
@@ -50,9 +50,6 @@ func generateFingerprint(hostname, cpuModel string) string {
|
|||||||
if info, err := cpu.Info(); err == nil && len(info) > 0 {
|
if info, err := cpu.Info(); err == nil && len(info) > 0 {
|
||||||
cpuModel = info[0].ModelName
|
cpuModel = info[0].ModelName
|
||||||
}
|
}
|
||||||
if cpuModel == "" {
|
|
||||||
cpuModel = getCpuModelFromCpuinfo()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
fingerprint = hostname + cpuModel
|
fingerprint = hostname + cpuModel
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-8
@@ -361,16 +361,12 @@ func (gm *GPUManager) calculateGPUAverage(id string, gpu *system.GPUData, cacheK
|
|||||||
|
|
||||||
// If no new data arrived
|
// If no new data arrived
|
||||||
if deltaCount == 0 {
|
if deltaCount == 0 {
|
||||||
// Only discrete GPUs report temp/memory, so treat all-zero as suspended (return zeros).
|
// If GPU appears suspended (instantaneous values are 0), return zero values
|
||||||
// Engine-based (Intel) GPUs don't, so carry the last average forward across sample gaps.
|
// Otherwise return last known average for temporary collection gaps
|
||||||
if gpu.Engines == nil && gpu.Temperature == 0 && gpu.MemoryUsed == 0 {
|
if gpu.Temperature == 0 && gpu.MemoryUsed == 0 {
|
||||||
return system.GPUData{Name: gpu.Name}
|
return system.GPUData{Name: gpu.Name}
|
||||||
}
|
}
|
||||||
lastAvg := gm.lastAvgData[id] // zero value if not found
|
return gm.lastAvgData[id] // zero value if not found
|
||||||
if lastAvg.Name == "" {
|
|
||||||
lastAvg.Name = gpu.Name
|
|
||||||
}
|
|
||||||
return lastAvg
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate new average
|
// Calculate new average
|
||||||
|
|||||||
+2
-37
@@ -566,42 +566,6 @@ func TestGetCurrentData(t *testing.T) {
|
|||||||
assert.EqualValues(t, 2, gm.GpuDataMap["0"].Count, "Count should still be 2")
|
assert.EqualValues(t, 2, gm.GpuDataMap["0"].Count, "Count should still be 2")
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("carries Intel GPU average forward between samples", func(t *testing.T) {
|
|
||||||
// Intel GPUs report no temp/memory, so between-sample gaps (delta 0) must
|
|
||||||
// reuse the last average instead of returning zeros and blanking the chart.
|
|
||||||
gm := &GPUManager{
|
|
||||||
GpuDataMap: map[string]*system.GPUData{
|
|
||||||
"0": {
|
|
||||||
Name: "GPU",
|
|
||||||
Usage: 0, // derived from engines for Intel
|
|
||||||
Power: 200, // averages to 100 over 2 counts
|
|
||||||
PowerPkg: 60, // averages to 30 over 2 counts
|
|
||||||
Count: 2,
|
|
||||||
Engines: map[string]float64{
|
|
||||||
"Render/3D": 80, // averages to 40
|
|
||||||
"Video": 20, // averages to 10
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheKey := uint16(1000) // realtime cache key
|
|
||||||
|
|
||||||
// First collection - computes and stores averages
|
|
||||||
result1 := gm.GetCurrentData(cacheKey)
|
|
||||||
assert.InDelta(t, 100.0, result1["0"].Power, 0.01)
|
|
||||||
assert.InDelta(t, 30.0, result1["0"].PowerPkg, 0.01)
|
|
||||||
assert.InDelta(t, 40.0, result1["0"].Engines["Render/3D"], 0.01)
|
|
||||||
|
|
||||||
// Second collection with no new sample (count unchanged, temp/mem still 0).
|
|
||||||
// Must carry the last average forward rather than blanking to zero.
|
|
||||||
result2 := gm.GetCurrentData(cacheKey)
|
|
||||||
assert.Equal(t, "GPU", result2["0"].Name, "Name should be preserved")
|
|
||||||
assert.InDelta(t, 100.0, result2["0"].Power, 0.01, "Should reuse last average power, not 0")
|
|
||||||
assert.InDelta(t, 30.0, result2["0"].PowerPkg, 0.01, "Should reuse last average package power, not 0")
|
|
||||||
assert.InDelta(t, 40.0, result2["0"].Engines["Render/3D"], 0.01, "Should reuse last average engine usage")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("tracks separate averages per cache key", func(t *testing.T) {
|
t.Run("tracks separate averages per cache key", func(t *testing.T) {
|
||||||
gm := &GPUManager{
|
gm := &GPUManager{
|
||||||
GpuDataMap: map[string]*system.GPUData{
|
GpuDataMap: map[string]*system.GPUData{
|
||||||
@@ -1119,6 +1083,7 @@ func TestCalculateGPUAverage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
|
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
|
||||||
|
// Save original PATH
|
||||||
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
|
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -1212,7 +1177,7 @@ echo "[]"`
|
|||||||
{
|
{
|
||||||
name: "no gpu tools available",
|
name: "no gpu tools available",
|
||||||
setupCommands: func(_ string) error {
|
setupCommands: func(_ string) error {
|
||||||
// The subtest already restricts PATH to its empty temporary directory.
|
t.Setenv("PATH", "")
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
"github.com/fxamacker/cbor/v2"
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -52,8 +51,6 @@ func NewHandlerRegistry() *HandlerRegistry {
|
|||||||
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
||||||
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
||||||
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
||||||
registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
|
|
||||||
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
|
|
||||||
|
|
||||||
return registry
|
return registry
|
||||||
}
|
}
|
||||||
@@ -181,23 +178,6 @@ func (h *GetSmartDataHandler) Handle(hctx *HandlerContext) error {
|
|||||||
}, hctx.RequestID)
|
}, hctx.RequestID)
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
// GetZfsDataHandler handles ZFS detail data requests
|
|
||||||
type GetZfsDataHandler struct{}
|
|
||||||
|
|
||||||
func (h *GetZfsDataHandler) Handle(hctx *HandlerContext) error {
|
|
||||||
if hctx.Agent.storagePoolManager == nil {
|
|
||||||
return hctx.SendResponse(nil, hctx.RequestID)
|
|
||||||
}
|
|
||||||
var req common.ZfsDataRequest
|
|
||||||
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return hctx.SendResponse(hctx.Agent.storagePoolManager.GetDetail(req.Force), hctx.RequestID)
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////
|
||||||
////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////
|
||||||
////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////
|
||||||
@@ -225,21 +205,3 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
|
|||||||
|
|
||||||
return hctx.SendResponse(details, hctx.RequestID)
|
return hctx.SendResponse(details, hctx.RequestID)
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
|
|
||||||
type SyncNetworkMonitorsHandler struct{}
|
|
||||||
|
|
||||||
func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
|
|
||||||
var req monitor.SyncRequest
|
|
||||||
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return hctx.SendResponse(resp, hctx.RequestID)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,10 +4,8 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
"github.com/fxamacker/cbor/v2"
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -32,32 +30,6 @@ func TestNewAgentResponseSmartData(t *testing.T) {
|
|||||||
assert.True(t, response.SmartComplete)
|
assert.True(t, response.SmartComplete)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetZfsDataHandlerForceRefresh(t *testing.T) {
|
|
||||||
poolCalls := 0
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
poolCalls++
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
zm.GetDetail(false)
|
|
||||||
|
|
||||||
requestData, err := cbor.Marshal(common.ZfsDataRequest{Force: true})
|
|
||||||
assert.NoError(t, err)
|
|
||||||
ctx := &HandlerContext{
|
|
||||||
Agent: &Agent{storagePoolManager: zm},
|
|
||||||
Request: &common.HubRequest[cbor.RawMessage]{
|
|
||||||
Action: common.GetZfsData,
|
|
||||||
Data: requestData,
|
|
||||||
},
|
|
||||||
SendResponse: func(any, *uint32) error { return nil },
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NoError(t, (&GetZfsDataHandler{}).Handle(ctx))
|
|
||||||
assert.Equal(t, 2, poolCalls)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *MockHandler) Handle(ctx *HandlerContext) error {
|
func (m *MockHandler) Handle(ctx *HandlerContext) error {
|
||||||
if m.handleFunc != nil {
|
if m.handleFunc != nil {
|
||||||
return m.handleFunc(ctx)
|
return m.handleFunc(ctx)
|
||||||
|
|||||||
+9
-64
@@ -17,17 +17,15 @@ import (
|
|||||||
var mdraidSysfsRoot = "/sys"
|
var mdraidSysfsRoot = "/sys"
|
||||||
|
|
||||||
type mdraidHealth struct {
|
type mdraidHealth struct {
|
||||||
level string
|
level string
|
||||||
arrayState string
|
arrayState string
|
||||||
degraded uint64
|
degraded uint64
|
||||||
faultyDisks uint64
|
raidDisks uint64
|
||||||
populatedDisks uint64
|
syncAction string
|
||||||
raidDisks uint64
|
syncCompleted string
|
||||||
syncAction string
|
syncSpeed string
|
||||||
syncCompleted string
|
mismatchCnt uint64
|
||||||
syncSpeed string
|
capacity uint64
|
||||||
mismatchCnt uint64
|
|
||||||
capacity uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// scanMdraidDevices discovers Linux md arrays exposed in sysfs.
|
// scanMdraidDevices discovers Linux md arrays exposed in sysfs.
|
||||||
@@ -94,9 +92,6 @@ func (sm *SmartManager) collectMdraidHealth(deviceInfo *DeviceInfo) (bool, error
|
|||||||
if health.degraded > 0 {
|
if health.degraded > 0 {
|
||||||
attrs = append(attrs, &smart.SmartAttribute{Name: "Degraded", RawValue: health.degraded})
|
attrs = append(attrs, &smart.SmartAttribute{Name: "Degraded", RawValue: health.degraded})
|
||||||
}
|
}
|
||||||
if health.faultyDisks > 0 {
|
|
||||||
attrs = append(attrs, &smart.SmartAttribute{Name: "FaultyDisks", RawValue: health.faultyDisks})
|
|
||||||
}
|
|
||||||
if health.syncAction != "" {
|
if health.syncAction != "" {
|
||||||
attrs = append(attrs, &smart.SmartAttribute{Name: "SyncAction", RawString: health.syncAction})
|
attrs = append(attrs, &smart.SmartAttribute{Name: "SyncAction", RawString: health.syncAction})
|
||||||
}
|
}
|
||||||
@@ -157,7 +152,6 @@ func readMdraidHealth(blockName string) (mdraidHealth, bool) {
|
|||||||
if val, ok := utils.ReadUintFile(filepath.Join(mdDir, "degraded")); ok {
|
if val, ok := utils.ReadUintFile(filepath.Join(mdDir, "degraded")); ok {
|
||||||
out.degraded = val
|
out.degraded = val
|
||||||
}
|
}
|
||||||
out.faultyDisks, out.populatedDisks = countMdraidMemberStates(blockName, mdraidSysfsRoot)
|
|
||||||
if val, ok := utils.ReadUintFile(filepath.Join(mdDir, "mismatch_cnt")); ok {
|
if val, ok := utils.ReadUintFile(filepath.Join(mdDir, "mismatch_cnt")); ok {
|
||||||
out.mismatchCnt = val
|
out.mismatchCnt = val
|
||||||
}
|
}
|
||||||
@@ -183,19 +177,7 @@ func mdraidSmartStatus(health mdraidHealth) string {
|
|||||||
case "resync", "recover", "reshape":
|
case "resync", "recover", "reshape":
|
||||||
return "WARNING"
|
return "WARNING"
|
||||||
}
|
}
|
||||||
// Use actual faulty member count rather than the degraded counter, which
|
|
||||||
// equals raid_disks minus active_disks. On QNAP systems raid_disks may be
|
|
||||||
// set to a large value (e.g. 32) while only a few slots are ever used,
|
|
||||||
// making degraded misleadingly large despite zero failed disks.
|
|
||||||
if health.faultyDisks > 0 {
|
|
||||||
return "FAILED"
|
|
||||||
}
|
|
||||||
if health.degraded > 0 {
|
if health.degraded > 0 {
|
||||||
if isSparseSlotDegraded(health) {
|
|
||||||
// A sysfs snapshot cannot distinguish reserved slots from a removed
|
|
||||||
// member on sparse arrays, so report the ambiguity as a warning.
|
|
||||||
return "WARNING"
|
|
||||||
}
|
|
||||||
return "FAILED"
|
return "FAILED"
|
||||||
}
|
}
|
||||||
if health.mismatchCnt > 0 {
|
if health.mismatchCnt > 0 {
|
||||||
@@ -214,43 +196,6 @@ func mdraidSmartStatus(health mdraidHealth) string {
|
|||||||
return "UNKNOWN"
|
return "UNKNOWN"
|
||||||
}
|
}
|
||||||
|
|
||||||
// countMdraidMemberStates reads member device directories under
|
|
||||||
// block/<name>/md and returns how many are explicitly marked "faulty", plus
|
|
||||||
// how many are populated at all (regardless of state). populatedDisks lets
|
|
||||||
// callers distinguish RAID slots that were never used (QNAP reserves far
|
|
||||||
// more raid_disks than it ever populates) from members that went missing.
|
|
||||||
func countMdraidMemberStates(blockName, root string) (faultyDisks, populatedDisks uint64) {
|
|
||||||
devDir := filepath.Join(root, "block", blockName, "md")
|
|
||||||
entries, err := os.ReadDir(devDir)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0
|
|
||||||
}
|
|
||||||
for _, ent := range entries {
|
|
||||||
if !strings.HasPrefix(ent.Name(), "dev-") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
populatedDisks++
|
|
||||||
statePath := filepath.Join(devDir, ent.Name(), "state")
|
|
||||||
state := utils.ReadStringFile(statePath)
|
|
||||||
if strings.Contains(state, "faulty") {
|
|
||||||
faultyDisks++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return faultyDisks, populatedDisks
|
|
||||||
}
|
|
||||||
|
|
||||||
// isSparseSlotDegraded reports whether a non-zero "degraded" count may be
|
|
||||||
// explained by RAID slots that were never populated. QNAP configures system
|
|
||||||
// arrays with raid_disks set to a large fixed maximum (e.g. 32) far beyond the
|
|
||||||
// handful of slots it ever populates, so sparse slots outnumber populated ones.
|
|
||||||
func isSparseSlotDegraded(health mdraidHealth) bool {
|
|
||||||
if health.populatedDisks == 0 || health.raidDisks <= health.populatedDisks {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
sparseSlots := health.raidDisks - health.populatedDisks
|
|
||||||
return sparseSlots > health.populatedDisks
|
|
||||||
}
|
|
||||||
|
|
||||||
// isMdraidBlockName matches /dev/mdN-style block device names.
|
// isMdraidBlockName matches /dev/mdN-style block device names.
|
||||||
func isMdraidBlockName(name string) bool {
|
func isMdraidBlockName(name string) bool {
|
||||||
if !strings.HasPrefix(name, "md") {
|
if !strings.HasPrefix(name, "md") {
|
||||||
|
|||||||
@@ -40,15 +40,6 @@ func TestMdraidMockSysfsScanAndCollect(t *testing.T) {
|
|||||||
write(filepath.Join(mdDir, "sync_completed"), "10%\n")
|
write(filepath.Join(mdDir, "sync_completed"), "10%\n")
|
||||||
write(filepath.Join(mdDir, "sync_speed"), "100M\n")
|
write(filepath.Join(mdDir, "sync_speed"), "100M\n")
|
||||||
write(filepath.Join(mdDir, "mismatch_cnt"), "0\n")
|
write(filepath.Join(mdDir, "mismatch_cnt"), "0\n")
|
||||||
|
|
||||||
// Simulate two healthy member devices (no faulty state).
|
|
||||||
for _, dev := range []string{"dev-sda", "dev-sdb"} {
|
|
||||||
devPath := filepath.Join(mdDir, dev)
|
|
||||||
if err := os.MkdirAll(devPath, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
write(filepath.Join(devPath, "state"), "in_sync\n")
|
|
||||||
}
|
|
||||||
write(filepath.Join(queueDir, "logical_block_size"), "512\n")
|
write(filepath.Join(queueDir, "logical_block_size"), "512\n")
|
||||||
write(filepath.Join(tmp, "block", "md0", "size"), "2048\n")
|
write(filepath.Join(tmp, "block", "md0", "size"), "2048\n")
|
||||||
|
|
||||||
@@ -90,77 +81,15 @@ func TestMdraidMockSysfsScanAndCollect(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCountMdraidMemberStates(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
|
|
||||||
write := func(path, content string) {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
mdDir := filepath.Join(tmp, "block", "md0", "md")
|
|
||||||
|
|
||||||
// No dev-* entries: zero faulty, zero populated.
|
|
||||||
if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 0 || populated != 0 {
|
|
||||||
t.Fatalf("no members: got (faulty=%d populated=%d), want (0,0)", faulty, populated)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Two healthy members.
|
|
||||||
write(filepath.Join(mdDir, "dev-sda", "state"), "in_sync\n")
|
|
||||||
write(filepath.Join(mdDir, "dev-sdb", "state"), "in_sync\n")
|
|
||||||
if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 0 || populated != 2 {
|
|
||||||
t.Fatalf("all in_sync: got (faulty=%d populated=%d), want (0,2)", faulty, populated)
|
|
||||||
}
|
|
||||||
|
|
||||||
// One faulty member.
|
|
||||||
write(filepath.Join(mdDir, "dev-sdb", "state"), "faulty\n")
|
|
||||||
if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 1 || populated != 2 {
|
|
||||||
t.Fatalf("one faulty: got (faulty=%d populated=%d), want (1,2)", faulty, populated)
|
|
||||||
}
|
|
||||||
|
|
||||||
// QNAP-style: 28 degraded slots but no dev-* entries for them, 4 in_sync.
|
|
||||||
write(filepath.Join(mdDir, "dev-sdb", "state"), "in_sync\n")
|
|
||||||
write(filepath.Join(mdDir, "dev-sdc", "state"), "in_sync\n")
|
|
||||||
write(filepath.Join(mdDir, "dev-sdd", "state"), "in_sync\n")
|
|
||||||
if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 0 || populated != 4 {
|
|
||||||
t.Fatalf("qnap sparse: got (faulty=%d populated=%d), want (0,4)", faulty, populated)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMdraidSmartStatus(t *testing.T) {
|
func TestMdraidSmartStatus(t *testing.T) {
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "inactive"}); got != "FAILED" {
|
if got := mdraidSmartStatus(mdraidHealth{arrayState: "inactive"}); got != "FAILED" {
|
||||||
t.Fatalf("mdraidSmartStatus(inactive) = %q, want FAILED", got)
|
t.Fatalf("mdraidSmartStatus(inactive) = %q, want FAILED", got)
|
||||||
}
|
}
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1, faultyDisks: 1, syncAction: "recover"}); got != "WARNING" {
|
if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1, syncAction: "recover"}); got != "WARNING" {
|
||||||
t.Fatalf("mdraidSmartStatus(degraded+recover) = %q, want WARNING", got)
|
t.Fatalf("mdraidSmartStatus(degraded+recover) = %q, want WARNING", got)
|
||||||
}
|
}
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1, faultyDisks: 1}); got != "FAILED" {
|
if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1}); got != "FAILED" {
|
||||||
t.Fatalf("mdraidSmartStatus(degraded+faulty) = %q, want FAILED", got)
|
t.Fatalf("mdraidSmartStatus(degraded) = %q, want FAILED", got)
|
||||||
}
|
|
||||||
// QNAP-style: raid_disks=32 but only 4 populated; degraded=28 but no faulty devices.
|
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 28, faultyDisks: 0, raidDisks: 32, populatedDisks: 4}); got != "WARNING" {
|
|
||||||
t.Fatalf("mdraidSmartStatus(qnap sparse) = %q, want WARNING", got)
|
|
||||||
}
|
|
||||||
// A member disappearing from the same sparse array is indistinguishable
|
|
||||||
// from another reserved slot, so it must not be reported as healthy.
|
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 29, faultyDisks: 0, raidDisks: 32, populatedDisks: 3}); got != "WARNING" {
|
|
||||||
t.Fatalf("mdraidSmartStatus(qnap sparse missing member) = %q, want WARNING", got)
|
|
||||||
}
|
|
||||||
// A genuinely missing member (removed dev-* entry, not just an unpopulated
|
|
||||||
// QNAP reserve slot) must still fail: raid_disks=4, only 3 populated, all
|
|
||||||
// of them in_sync, so faultyDisks==0 but degraded==1.
|
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 1, faultyDisks: 0, raidDisks: 4, populatedDisks: 3}); got != "FAILED" {
|
|
||||||
t.Fatalf("mdraidSmartStatus(missing member) = %q, want FAILED", got)
|
|
||||||
}
|
|
||||||
// Degraded with no member-state info at all (e.g. sysfs read failed) must
|
|
||||||
// still fail rather than being silently treated as a sparse QNAP array.
|
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 1, faultyDisks: 0, raidDisks: 4, populatedDisks: 0}); got != "FAILED" {
|
|
||||||
t.Fatalf("mdraidSmartStatus(degraded, no member info) = %q, want FAILED", got)
|
|
||||||
}
|
}
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", syncAction: "recover"}); got != "WARNING" {
|
if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", syncAction: "recover"}); got != "WARNING" {
|
||||||
t.Fatalf("mdraidSmartStatus(recover) = %q, want WARNING", got)
|
t.Fatalf("mdraidSmartStatus(recover) = %q, want WARNING", got)
|
||||||
|
|||||||
@@ -1,176 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MonitorManager manages network monitor configurations and task lifetimes.
|
|
||||||
type MonitorManager struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
monitors map[string]*monitorTask // keyed by monitor ID
|
|
||||||
probe monitorProbe
|
|
||||||
resumeGuard monitorResumeGuard
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorManager() *MonitorManager {
|
|
||||||
return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
|
|
||||||
return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
|
|
||||||
}
|
|
||||||
|
|
||||||
// SyncMonitors replaces all monitor tasks with the given configs.
|
|
||||||
func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
|
|
||||||
pm.mu.Lock()
|
|
||||||
defer pm.mu.Unlock()
|
|
||||||
|
|
||||||
// Build set of new keys
|
|
||||||
newKeys := make(map[string]monitor.Config, len(configs))
|
|
||||||
for _, cfg := range configs {
|
|
||||||
if cfg.ID == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
newKeys[cfg.ID] = cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop removed monitors
|
|
||||||
for key, task := range pm.monitors {
|
|
||||||
if _, exists := newKeys[key]; !exists {
|
|
||||||
task.cancel()
|
|
||||||
delete(pm.monitors, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start new monitors and restart tasks whose config changed.
|
|
||||||
for key, cfg := range newKeys {
|
|
||||||
task, exists := pm.monitors[key]
|
|
||||||
if exists && task.config == cfg {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
task.cancel()
|
|
||||||
}
|
|
||||||
task = newMonitorTaskFromExisting(cfg, task)
|
|
||||||
task.resumeGuard = &pm.resumeGuard
|
|
||||||
pm.resumeGuard.start()
|
|
||||||
pm.monitors[key] = task
|
|
||||||
pm.startMonitor(task)
|
|
||||||
}
|
|
||||||
if len(pm.monitors) == 0 {
|
|
||||||
pm.resumeGuard.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleSyncRequest applies a full or incremental monitor sync request.
|
|
||||||
func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
|
|
||||||
switch req.Action {
|
|
||||||
case monitor.SyncActionReplace:
|
|
||||||
pm.SyncMonitors(req.Configs)
|
|
||||||
return monitor.SyncResponse{}, nil
|
|
||||||
case monitor.SyncActionUpsert:
|
|
||||||
result, err := pm.UpsertMonitor(req.Config, req.RunNow)
|
|
||||||
if err != nil {
|
|
||||||
return monitor.SyncResponse{}, err
|
|
||||||
}
|
|
||||||
if result == nil {
|
|
||||||
return monitor.SyncResponse{}, nil
|
|
||||||
}
|
|
||||||
return monitor.SyncResponse{Result: *result}, nil
|
|
||||||
case monitor.SyncActionDelete:
|
|
||||||
if req.Config.ID == "" {
|
|
||||||
return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
|
|
||||||
}
|
|
||||||
pm.DeleteMonitor(req.Config.ID)
|
|
||||||
return monitor.SyncResponse{}, nil
|
|
||||||
default:
|
|
||||||
return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpsertMonitor creates or replaces a single monitor task.
|
|
||||||
func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
|
|
||||||
if config.ID == "" {
|
|
||||||
return nil, errors.New("missing monitor ID")
|
|
||||||
}
|
|
||||||
|
|
||||||
pm.mu.Lock()
|
|
||||||
task, exists := pm.monitors[config.ID]
|
|
||||||
if exists && task.config == config {
|
|
||||||
pm.mu.Unlock()
|
|
||||||
if !runNow {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return task.runProbe(pm.probe), nil
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
task.cancel()
|
|
||||||
}
|
|
||||||
task = newMonitorTaskFromExisting(config, task)
|
|
||||||
task.resumeGuard = &pm.resumeGuard
|
|
||||||
pm.resumeGuard.start()
|
|
||||||
pm.monitors[config.ID] = task
|
|
||||||
pm.mu.Unlock()
|
|
||||||
|
|
||||||
if runNow {
|
|
||||||
result := task.runProbe(pm.probe)
|
|
||||||
pm.startMonitor(task)
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
pm.startMonitor(task)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteMonitor stops and removes a single monitor task.
|
|
||||||
func (pm *MonitorManager) DeleteMonitor(id string) {
|
|
||||||
if id == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
pm.mu.Lock()
|
|
||||||
defer pm.mu.Unlock()
|
|
||||||
if task, exists := pm.monitors[id]; exists {
|
|
||||||
task.cancel()
|
|
||||||
delete(pm.monitors, id)
|
|
||||||
}
|
|
||||||
if len(pm.monitors) == 0 {
|
|
||||||
pm.resumeGuard.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
|
|
||||||
func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
|
|
||||||
pm.mu.RLock()
|
|
||||||
defer pm.mu.RUnlock()
|
|
||||||
|
|
||||||
results := make(map[string]monitor.Result, len(pm.monitors))
|
|
||||||
now := time.Now()
|
|
||||||
duration := time.Duration(durationMs) * time.Millisecond
|
|
||||||
|
|
||||||
for _, task := range pm.monitors {
|
|
||||||
result, ok := task.history.result(duration, now)
|
|
||||||
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
results[task.config.ID] = result
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop stops all monitor tasks.
|
|
||||||
func (pm *MonitorManager) Stop() {
|
|
||||||
pm.mu.Lock()
|
|
||||||
defer pm.mu.Unlock()
|
|
||||||
for key, task := range pm.monitors {
|
|
||||||
task.cancel()
|
|
||||||
delete(pm.monitors, key)
|
|
||||||
}
|
|
||||||
pm.resumeGuard.shutdown()
|
|
||||||
}
|
|
||||||
@@ -1,274 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Monitors run at user-defined intervals (e.g., every 10s).
|
|
||||||
// To keep memory usage low and constant, data is stored in two layers:
|
|
||||||
// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
|
|
||||||
// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
|
|
||||||
// wall-clock minute. Samples collected within the same minute are aggregated
|
|
||||||
// (sum, min, max, count) into a single bucket.
|
|
||||||
//
|
|
||||||
// Short-term requests (<= 61s) use raw samples.
|
|
||||||
// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
|
|
||||||
// of individual data points.
|
|
||||||
|
|
||||||
const (
|
|
||||||
// monitorRawRetention is the duration to keep individual samples
|
|
||||||
monitorRawRetention = 61 * time.Second
|
|
||||||
// monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
|
|
||||||
monitorMinuteBucketLen int32 = 61
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorHistory owns retention and aggregation, independently of probe execution.
|
|
||||||
type monitorHistory struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
sampleCount int64
|
|
||||||
samples []monitorSample
|
|
||||||
buckets [monitorMinuteBucketLen]monitorBucket
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorHistory() *monitorHistory {
|
|
||||||
// Start small for typical intervals; append grows the buffer for faster probes.
|
|
||||||
return &monitorHistory{samples: make([]monitorSample, 0, 4)}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorHistory) clone() *monitorHistory {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
cloned := newMonitorHistory()
|
|
||||||
cloned.samples = append(cloned.samples, h.samples...)
|
|
||||||
cloned.buckets = h.buckets
|
|
||||||
cloned.sampleCount = h.sampleCount
|
|
||||||
return cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
return h.resultLocked(duration, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorHistory) record(sample monitorSample) monitor.Result {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
h.addSampleLocked(sample)
|
|
||||||
result, _ := h.resultLocked(time.Minute, sample.timestamp)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorSample stores one monitor attempt and its collection time.
|
|
||||||
type monitorSample struct {
|
|
||||||
responseUs int64 // -1 means loss
|
|
||||||
timestamp time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorBucket stores one minute of aggregated monitor data.
|
|
||||||
type monitorBucket struct {
|
|
||||||
minute int32
|
|
||||||
filled bool
|
|
||||||
stats monitorAggregate
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorAggregate accumulates successful response stats and total sample counts.
|
|
||||||
type monitorAggregate struct {
|
|
||||||
sumUs int64
|
|
||||||
minUs int64
|
|
||||||
maxUs int64
|
|
||||||
totalCount int64
|
|
||||||
successCount int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// newMonitorAggregate initializes an aggregate with an unset minimum value.
|
|
||||||
func newMonitorAggregate() monitorAggregate {
|
|
||||||
return monitorAggregate{minUs: math.MaxInt64}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addResponse folds a single monitor sample into the aggregate.
|
|
||||||
func (agg *monitorAggregate) addResponse(responseUs int64) {
|
|
||||||
agg.totalCount++
|
|
||||||
if responseUs < 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
agg.successCount++
|
|
||||||
agg.sumUs += responseUs
|
|
||||||
if responseUs < agg.minUs {
|
|
||||||
agg.minUs = responseUs
|
|
||||||
}
|
|
||||||
if responseUs > agg.maxUs {
|
|
||||||
agg.maxUs = responseUs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAggregate merges another aggregate into this one.
|
|
||||||
func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
|
|
||||||
if other.totalCount == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
agg.totalCount += other.totalCount
|
|
||||||
agg.successCount += other.successCount
|
|
||||||
agg.sumUs += other.sumUs
|
|
||||||
if other.successCount == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
|
|
||||||
agg.minUs = other.minUs
|
|
||||||
}
|
|
||||||
if other.maxUs > agg.maxUs {
|
|
||||||
agg.maxUs = other.maxUs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// hasData reports whether the aggregate contains any samples.
|
|
||||||
func (agg monitorAggregate) hasData() bool {
|
|
||||||
return agg.totalCount > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// result converts the aggregate into the monitor result format.
|
|
||||||
func (agg monitorAggregate) result() monitor.Result {
|
|
||||||
avg := agg.avgResponse()
|
|
||||||
result := monitor.Result{
|
|
||||||
AvgResponse: avg,
|
|
||||||
MinResponse: agg.minUs,
|
|
||||||
MaxResponse: agg.maxUs,
|
|
||||||
PacketLoss: agg.lossPercentage(),
|
|
||||||
TotalCount: agg.totalCount,
|
|
||||||
SuccessCount: agg.successCount,
|
|
||||||
ResponseSum: agg.sumUs,
|
|
||||||
}
|
|
||||||
if agg.successCount == 0 {
|
|
||||||
result.MinResponse, result.MaxResponse = 0, 0
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// avgResponse returns the rounded average of successful samples.
|
|
||||||
func (agg monitorAggregate) avgResponse() int64 {
|
|
||||||
if agg.successCount == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return agg.sumUs / agg.successCount
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// lossPercentage returns the rounded failure rate for the aggregate.
|
|
||||||
func (agg monitorAggregate) lossPercentage() float64 {
|
|
||||||
if agg.totalCount == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
|
|
||||||
}
|
|
||||||
|
|
||||||
// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
|
|
||||||
func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
|
|
||||||
agg := h.aggregateLocked(duration, now)
|
|
||||||
if !agg.hasData() {
|
|
||||||
// short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
|
|
||||||
// between monitor samples since monitors run at longer, user-defined intervals; fall back to
|
|
||||||
// the most recent sample so realtime requests still report current status.
|
|
||||||
agg = h.latestSampleAggregateLocked()
|
|
||||||
}
|
|
||||||
hourAgg := h.aggregateLocked(time.Hour, now)
|
|
||||||
if !agg.hasData() {
|
|
||||||
return monitor.Result{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
result := agg.result()
|
|
||||||
if len(h.samples) > 0 {
|
|
||||||
result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
|
|
||||||
}
|
|
||||||
|
|
||||||
result.AvgResponse1h = hourAgg.avgResponse()
|
|
||||||
result.MinResponse1h = hourAgg.minUs
|
|
||||||
result.MaxResponse1h = hourAgg.maxUs
|
|
||||||
result.PacketLoss1h = hourAgg.lossPercentage()
|
|
||||||
result.SampleCount = h.sampleCount
|
|
||||||
|
|
||||||
if hourAgg.successCount == 0 {
|
|
||||||
result.MinResponse1h, result.MaxResponse1h = 0, 0
|
|
||||||
}
|
|
||||||
return result, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
|
|
||||||
func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
|
|
||||||
agg := newMonitorAggregate()
|
|
||||||
if len(h.samples) == 0 {
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
agg.addResponse(h.samples[len(h.samples)-1].responseUs)
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregateLocked collects monitor data for the requested time window.
|
|
||||||
func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
|
|
||||||
cutoff := now.Add(-duration)
|
|
||||||
// Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
|
|
||||||
if duration <= monitorRawRetention {
|
|
||||||
return aggregateSamplesSince(h.samples, cutoff)
|
|
||||||
}
|
|
||||||
return aggregateBucketsSince(h.buckets[:], cutoff, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregateSamplesSince aggregates raw samples newer than the cutoff.
|
|
||||||
func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
|
|
||||||
agg := newMonitorAggregate()
|
|
||||||
for _, sample := range samples {
|
|
||||||
if sample.timestamp.Before(cutoff) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
agg.addResponse(sample.responseUs)
|
|
||||||
}
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
|
|
||||||
func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
|
|
||||||
agg := newMonitorAggregate()
|
|
||||||
startMinute := int32(cutoff.Unix() / 60)
|
|
||||||
endMinute := int32(now.Unix() / 60)
|
|
||||||
for _, bucket := range buckets {
|
|
||||||
if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
agg.addAggregate(bucket.stats)
|
|
||||||
}
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
|
|
||||||
// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
|
|
||||||
func (h *monitorHistory) addSampleLocked(sample monitorSample) {
|
|
||||||
h.sampleCount++
|
|
||||||
cutoff := sample.timestamp.Add(-monitorRawRetention)
|
|
||||||
start := 0
|
|
||||||
for i := range h.samples {
|
|
||||||
if !h.samples[i].timestamp.Before(cutoff) {
|
|
||||||
start = i
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if i == len(h.samples)-1 {
|
|
||||||
start = len(h.samples)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if start > 0 {
|
|
||||||
size := copy(h.samples, h.samples[start:])
|
|
||||||
h.samples = h.samples[:size]
|
|
||||||
}
|
|
||||||
h.samples = append(h.samples, sample)
|
|
||||||
|
|
||||||
minute := int32(sample.timestamp.Unix() / 60)
|
|
||||||
// Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
|
|
||||||
bucket := &h.buckets[minute%monitorMinuteBucketLen]
|
|
||||||
if !bucket.filled || bucket.minute != minute {
|
|
||||||
bucket.minute = minute
|
|
||||||
bucket.filled = true
|
|
||||||
bucket.stats = newMonitorAggregate()
|
|
||||||
}
|
|
||||||
bucket.stats.addResponse(sample.responseUs)
|
|
||||||
}
|
|
||||||
@@ -1,154 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorHistoryWindowCounts(t *testing.T) {
|
|
||||||
history := newMonitorHistory()
|
|
||||||
now := time.Now()
|
|
||||||
// This older success counts toward lifetime warm-up, but not this window.
|
|
||||||
history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
|
|
||||||
history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
|
|
||||||
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
|
||||||
result, ok := history.result(time.Minute, now)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.EqualValues(t, 4, result.SampleCount)
|
|
||||||
assert.EqualValues(t, 3, result.TotalCount)
|
|
||||||
assert.EqualValues(t, 2, result.SuccessCount)
|
|
||||||
assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
|
|
||||||
assert.EqualValues(t, 15, result.AvgResponse)
|
|
||||||
assert.Equal(t, 33.33, result.PacketLoss)
|
|
||||||
|
|
||||||
encoded, err := cbor.Marshal(result)
|
|
||||||
require.NoError(t, err)
|
|
||||||
var decoded monitor.Result
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
|
|
||||||
assert.Equal(t, result, decoded)
|
|
||||||
stats := monitor.Stats{}.FromResult(decoded)
|
|
||||||
assert.Equal(t, result.TotalCount, stats.TotalCount)
|
|
||||||
assert.Equal(t, result.SuccessCount, stats.SuccessCount)
|
|
||||||
assert.Equal(t, result.ResponseSum, stats.ResponseSum)
|
|
||||||
|
|
||||||
// Reads do not consume samples. A short window's latest-sample fallback
|
|
||||||
// carries the count for that single failure, not the minute or lifetime count.
|
|
||||||
repeated, _ := history.result(time.Minute, now)
|
|
||||||
assert.Equal(t, result, repeated)
|
|
||||||
fallback, ok := history.result(time.Second, now)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.EqualValues(t, 1, fallback.TotalCount)
|
|
||||||
assert.Zero(t, fallback.SuccessCount)
|
|
||||||
assert.Zero(t, fallback.ResponseSum)
|
|
||||||
assert.Equal(t, 100.0, fallback.PacketLoss)
|
|
||||||
assert.EqualValues(t, 4, fallback.SampleCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
|
|
||||||
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
|
|
||||||
history := newMonitorHistory()
|
|
||||||
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
|
||||||
|
|
||||||
agg := history.aggregateLocked(time.Minute, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(2), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(1), agg.successCount)
|
|
||||||
result := agg.result()
|
|
||||||
assert.Equal(t, int64(20), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MaxResponse)
|
|
||||||
assert.Equal(t, 50.0, result.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
|
|
||||||
now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
|
|
||||||
history := newMonitorHistory()
|
|
||||||
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
|
|
||||||
agg := history.aggregateLocked(10*time.Minute, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(4), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(3), agg.successCount)
|
|
||||||
result := agg.result()
|
|
||||||
assert.Equal(t, int64(30), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(40), result.MaxResponse)
|
|
||||||
assert.Equal(t, 25.0, result.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
|
|
||||||
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
|
|
||||||
history := newMonitorHistory()
|
|
||||||
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
|
|
||||||
|
|
||||||
require.Len(t, history.samples, 1)
|
|
||||||
assert.Equal(t, int64(20), history.samples[0].responseUs)
|
|
||||||
|
|
||||||
agg := history.aggregateLocked(10*time.Minute, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(2), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(2), agg.successCount)
|
|
||||||
result := agg.result()
|
|
||||||
assert.Equal(t, int64(15), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(10), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MaxResponse)
|
|
||||||
assert.Equal(t, 0.0, result.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryProbeTimestamp(t *testing.T) {
|
|
||||||
history := newMonitorHistory()
|
|
||||||
start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
|
|
||||||
_, ok := history.result(time.Minute, start)
|
|
||||||
require.False(t, ok)
|
|
||||||
first := history.record(monitorSample{responseUs: 20, timestamp: start})
|
|
||||||
assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
|
|
||||||
for minute := 0; minute < 5; minute++ {
|
|
||||||
now := start.Add(time.Duration(minute)*time.Minute + time.Second)
|
|
||||||
// Realtime reads must not consume freshness for the persistence request.
|
|
||||||
for _, window := range []time.Duration{time.Second, time.Minute} {
|
|
||||||
result, ok := history.result(window, now)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
|
|
||||||
assert.Equal(t, int64(20), result.AvgResponse)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next := start.Add(5 * time.Minute)
|
|
||||||
failed := history.record(monitorSample{responseUs: -1, timestamp: next})
|
|
||||||
assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
|
|
||||||
assert.Equal(t, float64(100), failed.PacketLoss)
|
|
||||||
repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
|
|
||||||
assert.Equal(t, float64(100), repeated.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistorySampleCount(t *testing.T) {
|
|
||||||
history := newMonitorHistory()
|
|
||||||
now := time.Now()
|
|
||||||
// Both failed and successful probes count, including older samples so
|
|
||||||
// monitors with hourly intervals can finish warming up.
|
|
||||||
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
|
|
||||||
for i, response := range []int64{10, -1, 20} {
|
|
||||||
result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
|
|
||||||
assert.EqualValues(t, i+2, result.SampleCount)
|
|
||||||
}
|
|
||||||
result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.EqualValues(t, 4, result.SampleCount)
|
|
||||||
}
|
|
||||||
@@ -1,312 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"regexp"
|
|
||||||
"runtime"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"golang.org/x/net/icmp"
|
|
||||||
"golang.org/x/net/ipv4"
|
|
||||||
"golang.org/x/net/ipv6"
|
|
||||||
|
|
||||||
"log/slog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Match the numeric RTT independently of the localized label used by Windows.
|
|
||||||
var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
|
|
||||||
|
|
||||||
var icmpSequence atomic.Uint32
|
|
||||||
|
|
||||||
type icmpPacketConn interface {
|
|
||||||
Close() error
|
|
||||||
}
|
|
||||||
|
|
||||||
// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
|
|
||||||
// all native methods fail, the choice is cached so subsequent monitors skip
|
|
||||||
// the trial-and-error overhead.
|
|
||||||
type icmpMethod uint8
|
|
||||||
|
|
||||||
const (
|
|
||||||
icmpUntried icmpMethod = iota // haven't tried yet
|
|
||||||
icmpRaw // privileged raw socket
|
|
||||||
icmpDatagram // unprivileged datagram socket
|
|
||||||
icmpExecFallback // shell out to system ping command
|
|
||||||
)
|
|
||||||
|
|
||||||
// icmpFamily holds the network parameters and cached detection result for one address family.
|
|
||||||
type icmpFamily struct {
|
|
||||||
rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
|
|
||||||
dgramNetwork string // e.g. "udp4" or "udp6"
|
|
||||||
listenAddr string // "0.0.0.0" or "::"
|
|
||||||
echoType icmp.Type // outgoing echo request type
|
|
||||||
replyType icmp.Type // expected echo reply type
|
|
||||||
proto int // IANA protocol number for parsing replies
|
|
||||||
isIPv6 bool
|
|
||||||
mode icmpMethod // cached detection result (guarded by icmpModeMu)
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
icmpV4 = icmpFamily{
|
|
||||||
rawNetwork: "ip4:icmp",
|
|
||||||
dgramNetwork: "udp4",
|
|
||||||
listenAddr: "0.0.0.0",
|
|
||||||
echoType: ipv4.ICMPTypeEcho,
|
|
||||||
replyType: ipv4.ICMPTypeEchoReply,
|
|
||||||
proto: 1,
|
|
||||||
}
|
|
||||||
icmpV6 = icmpFamily{
|
|
||||||
rawNetwork: "ip6:ipv6-icmp",
|
|
||||||
dgramNetwork: "udp6",
|
|
||||||
listenAddr: "::",
|
|
||||||
echoType: ipv6.ICMPTypeEchoRequest,
|
|
||||||
replyType: ipv6.ICMPTypeEchoReply,
|
|
||||||
proto: 58,
|
|
||||||
isIPv6: true,
|
|
||||||
}
|
|
||||||
icmpModeMu sync.Mutex
|
|
||||||
icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
|
|
||||||
return icmp.ListenPacket(network, listenAddr)
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorICMP sends an ICMP echo request and measures round-trip response.
|
|
||||||
// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
|
|
||||||
// unprivileged datagram, or exec fallback) is detected once per address
|
|
||||||
// family and cached for subsequent monitors.
|
|
||||||
// Returns response in microseconds, or -1 and an error on failure.
|
|
||||||
func monitorICMP(ctx context.Context, target string) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
family, ip, err := resolveICMPTarget(ctx, target)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
icmpModeMu.Lock()
|
|
||||||
if family.mode == icmpUntried {
|
|
||||||
family.mode = detectICMPMode(family, icmpListen)
|
|
||||||
}
|
|
||||||
mode := family.mode
|
|
||||||
icmpModeMu.Unlock()
|
|
||||||
|
|
||||||
switch mode {
|
|
||||||
case icmpRaw:
|
|
||||||
return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
|
|
||||||
case icmpDatagram:
|
|
||||||
return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
|
|
||||||
case icmpExecFallback:
|
|
||||||
return monitorICMPExec(ctx, ip.String(), family.isIPv6)
|
|
||||||
default:
|
|
||||||
return -1, errors.New("unsupported ICMP mode")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveICMPTarget resolves a target hostname or IP to determine the address
|
|
||||||
// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
|
|
||||||
func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
|
|
||||||
if ip := net.ParseIP(target); ip != nil {
|
|
||||||
if ip.To4() != nil {
|
|
||||||
return &icmpV4, ip.To4(), nil
|
|
||||||
}
|
|
||||||
return &icmpV6, ip, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
|
|
||||||
if err != nil || len(ips) == 0 {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
for _, ip := range ips {
|
|
||||||
if v4 := ip.To4(); v4 != nil {
|
|
||||||
return &icmpV4, v4, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &icmpV6, ips[0], nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
|
|
||||||
label := "IPv4"
|
|
||||||
if family.isIPv6 {
|
|
||||||
label = "IPv6"
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err := listen(family.rawNetwork, family.listenAddr)
|
|
||||||
slog.Debug("ICMP raw socket test", "family", label, "err", err)
|
|
||||||
if err == nil {
|
|
||||||
conn.Close()
|
|
||||||
return icmpRaw
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err = listen(family.dgramNetwork, family.listenAddr)
|
|
||||||
slog.Debug("ICMP datagram socket test", "family", label, "err", err)
|
|
||||||
if err == nil {
|
|
||||||
conn.Close()
|
|
||||||
return icmpDatagram
|
|
||||||
}
|
|
||||||
|
|
||||||
return icmpExecFallback
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
|
|
||||||
func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
|
|
||||||
conn, err := icmp.ListenPacket(network, family.listenAddr)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
|
|
||||||
return monitorICMPPacket(ctx, conn, family, dst)
|
|
||||||
}
|
|
||||||
|
|
||||||
func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
// Closing the socket interrupts both reads and writes on cancellation.
|
|
||||||
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
// Prepare correlation data before starting the round-trip timer. The token
|
|
||||||
// also distinguishes delayed replies after the 16-bit sequence wraps.
|
|
||||||
token := make([]byte, 16)
|
|
||||||
if _, err := rand.Read(token); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
echo := &icmp.Echo{
|
|
||||||
ID: os.Getpid() & 0xffff,
|
|
||||||
Seq: int(icmpSequence.Add(1) & 0xffff),
|
|
||||||
Data: token,
|
|
||||||
}
|
|
||||||
// Linux ping sockets replace the Echo ID with their bound port. Darwin
|
|
||||||
// datagram sockets and raw sockets preserve the supplied ID.
|
|
||||||
if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
|
|
||||||
echo.ID = local.Port
|
|
||||||
}
|
|
||||||
targetIP := icmpAddrIP(dst)
|
|
||||||
msg := &icmp.Message{
|
|
||||||
Type: family.echoType,
|
|
||||||
Code: 0,
|
|
||||||
Body: echo,
|
|
||||||
}
|
|
||||||
msgBytes, err := msg.Marshal(nil)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set deadline before sending
|
|
||||||
if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
buf := make([]byte, 1500)
|
|
||||||
start := time.Now()
|
|
||||||
if _, err := conn.WriteTo(msgBytes, dst); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read reply
|
|
||||||
for {
|
|
||||||
n, peer, err := conn.ReadFrom(buf)
|
|
||||||
received := time.Now()
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
if !targetIP.Equal(icmpAddrIP(peer)) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
reply, err := icmp.ParseMessage(family.proto, buf[:n])
|
|
||||||
if err != nil || reply.Type != family.replyType || reply.Code != 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
body, ok := reply.Body.(*icmp.Echo)
|
|
||||||
if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
|
|
||||||
return received.Sub(start).Microseconds(), nil
|
|
||||||
}
|
|
||||||
// Keep waiting for our reply without extending the original deadline.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func icmpAddrIP(addr net.Addr) net.IP {
|
|
||||||
switch addr := addr.(type) {
|
|
||||||
case *net.IPAddr:
|
|
||||||
return addr.IP
|
|
||||||
case *net.UDPAddr:
|
|
||||||
return addr.IP
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pingCommand selects the executable and arguments for the supported agent platforms.
|
|
||||||
// The context deadline enforces the timeout: -W has incompatible meanings across
|
|
||||||
// Linux, BSD IPv4 ping, and macOS ping6.
|
|
||||||
func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
|
|
||||||
family := "-4"
|
|
||||||
if isIPv6 {
|
|
||||||
family = "-6"
|
|
||||||
}
|
|
||||||
switch goos {
|
|
||||||
case "windows":
|
|
||||||
return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
|
|
||||||
case "linux":
|
|
||||||
return "ping", []string{family, "-n", "-c", "1", target}, nil
|
|
||||||
case "darwin", "freebsd", "openbsd":
|
|
||||||
command := "ping"
|
|
||||||
if isIPv6 {
|
|
||||||
command = "ping6"
|
|
||||||
}
|
|
||||||
return command, []string{"-n", "-c", "1", target}, nil
|
|
||||||
default:
|
|
||||||
return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
|
|
||||||
func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
cmd := exec.CommandContext(ctx, name, args...)
|
|
||||||
// Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
|
|
||||||
cmd.Env = append(os.Environ(), "LC_ALL=C")
|
|
||||||
output, err := cmd.Output()
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return -1, ctx.Err()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return -1, fmt.Errorf("%s failed: %w", name, err)
|
|
||||||
}
|
|
||||||
return parsePingResponse(output)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parsePingResponse returns the reported RTT, never subprocess execution time.
|
|
||||||
// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
|
|
||||||
func parsePingResponse(output []byte) (int64, error) {
|
|
||||||
matches := pingTimeRegex.FindSubmatch(output)
|
|
||||||
if len(matches) < 2 {
|
|
||||||
return -1, errors.New("ping output contains no round-trip time")
|
|
||||||
}
|
|
||||||
ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
|
|
||||||
if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
|
|
||||||
return -1, errors.New("invalid round-trip time in ping output")
|
|
||||||
}
|
|
||||||
return int64(math.Round(ms * 1000)), nil
|
|
||||||
}
|
|
||||||
@@ -1,433 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"golang.org/x/net/icmp"
|
|
||||||
)
|
|
||||||
|
|
||||||
type testICMPPacketConn struct{}
|
|
||||||
|
|
||||||
func (testICMPPacketConn) Close() error { return nil }
|
|
||||||
|
|
||||||
type blockingICMPConn struct {
|
|
||||||
net.PacketConn
|
|
||||||
reading chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
|
|
||||||
return len(p), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
|
|
||||||
close(c.reading)
|
|
||||||
return c.PacketConn.ReadFrom(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPPacketCancellation(t *testing.T) {
|
|
||||||
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer conn.Close()
|
|
||||||
blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
_, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
|
|
||||||
done <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-blocking.reading:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("probe did not begin reading")
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.Error(t, err)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("cancellation did not interrupt the socket read")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPExecCancellation(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("test uses a POSIX shell stub for ping")
|
|
||||||
}
|
|
||||||
dir := t.TempDir()
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
|
|
||||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
_, err := monitorICMPExec(ctx, "127.0.0.1", false)
|
|
||||||
done <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("cancellation did not terminate ping")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPingCommand(t *testing.T) {
|
|
||||||
for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
|
|
||||||
for _, ipv6 := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
|
|
||||||
target, family := "192.0.2.1", "-4"
|
|
||||||
if ipv6 {
|
|
||||||
target, family = "2001:db8::1", "-6"
|
|
||||||
}
|
|
||||||
name, args, err := pingCommand(goos, target, ipv6)
|
|
||||||
require.NoError(t, err)
|
|
||||||
wantName := "ping"
|
|
||||||
wantArgs := []string{"-n", "-c", "1", target}
|
|
||||||
switch goos {
|
|
||||||
case "windows":
|
|
||||||
wantArgs = []string{family, "-n", "1", "-w", "3000", target}
|
|
||||||
case "linux":
|
|
||||||
wantArgs = append([]string{family}, wantArgs...)
|
|
||||||
default:
|
|
||||||
if ipv6 {
|
|
||||||
wantName = "ping6"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert.Equal(t, wantName, name)
|
|
||||||
assert.Equal(t, wantArgs, args)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_, _, err := pingCommand("unsupported", "192.0.2.1", false)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParsePingResponse(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
output string
|
|
||||||
wantUs int64
|
|
||||||
}{
|
|
||||||
{"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
|
|
||||||
{"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
|
|
||||||
{"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
|
|
||||||
{"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
|
|
||||||
{"windows submillisecond", "Reply from ::1: time<1ms", 1000},
|
|
||||||
{"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
|
|
||||||
{"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
|
|
||||||
{"rounding", "time=0.1236 ms", 124},
|
|
||||||
{"empty", "", -1},
|
|
||||||
{"timeout", "Request timed out.", -1},
|
|
||||||
{"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
|
|
||||||
{"malformed", "time=oops ms", -1},
|
|
||||||
{"negative", "time=-1 ms", -1},
|
|
||||||
{"overflow", "time=999999999999999999999 ms", -1},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
responseUs, err := parsePingResponse([]byte(tc.output))
|
|
||||||
if tc.wantUs < 0 {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
assert.Equal(t, tc.wantUs, responseUs)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPExecOutput(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("test uses a POSIX shell stub for ping")
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
output string
|
|
||||||
exit int
|
|
||||||
wantUs int64
|
|
||||||
}{
|
|
||||||
{"success", "time=1.234 ms", 0, 1234},
|
|
||||||
{"missing RTT", "unrecognized output", 0, -1},
|
|
||||||
{"failed command with RTT", "time=1.234 ms", 1, -1},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
// Also verify an inherited locale cannot override the C locale.
|
|
||||||
script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
|
|
||||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
||||||
t.Setenv("LC_ALL", "de_DE.UTF-8")
|
|
||||||
responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
|
|
||||||
if tc.wantUs < 0 {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
assert.Equal(t, tc.wantUs, responseUs)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type icmpTestReply struct {
|
|
||||||
data []byte
|
|
||||||
peer net.Addr
|
|
||||||
}
|
|
||||||
|
|
||||||
type scriptedICMPConn struct {
|
|
||||||
net.PacketConn
|
|
||||||
local net.Addr
|
|
||||||
onWrite func([]byte, net.Addr)
|
|
||||||
replies []icmpTestReply
|
|
||||||
reads int
|
|
||||||
deadlineSets int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
|
|
||||||
c.deadlineSets++
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
|
|
||||||
c.onWrite(data, dst)
|
|
||||||
return len(data), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
|
|
||||||
c.reads++
|
|
||||||
if len(c.replies) == 0 {
|
|
||||||
return 0, nil, os.ErrDeadlineExceeded
|
|
||||||
}
|
|
||||||
reply := c.replies[0]
|
|
||||||
c.replies = c.replies[1:]
|
|
||||||
return copy(buf, reply.data), reply.peer, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPReplyCorrelation(t *testing.T) {
|
|
||||||
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
|
|
||||||
for _, datagram := range []bool{false, true} {
|
|
||||||
network := family.rawNetwork
|
|
||||||
ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
|
|
||||||
if family.isIPv6 {
|
|
||||||
ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
|
|
||||||
}
|
|
||||||
var dst net.Addr = &net.IPAddr{IP: ip}
|
|
||||||
var wrongPeer net.Addr = &net.IPAddr{IP: other}
|
|
||||||
if datagram {
|
|
||||||
network = family.dgramNetwork
|
|
||||||
dst = &net.UDPAddr{IP: ip}
|
|
||||||
wrongPeer = &net.UDPAddr{IP: other}
|
|
||||||
}
|
|
||||||
for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
|
|
||||||
for _, eventuallyMatches := range []bool{false, true} {
|
|
||||||
ending := "timeout"
|
|
||||||
if eventuallyMatches {
|
|
||||||
ending = "success"
|
|
||||||
}
|
|
||||||
t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
|
|
||||||
conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
|
|
||||||
if datagram {
|
|
||||||
conn.local = &net.UDPAddr{Port: 12345}
|
|
||||||
if runtime.GOOS == "linux" {
|
|
||||||
// Deliberately differ from the process ID.
|
|
||||||
conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
conn.onWrite = func(data []byte, target net.Addr) {
|
|
||||||
require.Equal(t, dst, target)
|
|
||||||
request, err := icmp.ParseMessage(family.proto, data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
echo := request.Body.(*icmp.Echo)
|
|
||||||
expectedID := os.Getpid() & 0xffff
|
|
||||||
if datagram && runtime.GOOS == "linux" {
|
|
||||||
expectedID = conn.local.(*net.UDPAddr).Port
|
|
||||||
}
|
|
||||||
require.Equal(t, expectedID, echo.ID)
|
|
||||||
reply := &icmp.Message{Type: family.replyType, Body: echo}
|
|
||||||
valid, err := reply.Marshal(nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
peer := dst
|
|
||||||
switch mismatch {
|
|
||||||
case "source":
|
|
||||||
peer = wrongPeer
|
|
||||||
case "id":
|
|
||||||
echo.ID ^= 1
|
|
||||||
case "sequence":
|
|
||||||
echo.Seq ^= 1
|
|
||||||
case "payload":
|
|
||||||
echo.Data[0] ^= 1
|
|
||||||
case "type":
|
|
||||||
reply.Type = family.echoType
|
|
||||||
case "code":
|
|
||||||
reply.Code = 1
|
|
||||||
}
|
|
||||||
invalid, err := reply.Marshal(nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
if mismatch == "malformed" {
|
|
||||||
invalid = invalid[:2]
|
|
||||||
}
|
|
||||||
conn.replies = []icmpTestReply{{invalid, peer}}
|
|
||||||
if eventuallyMatches {
|
|
||||||
conn.replies = append(conn.replies, icmpTestReply{valid, dst})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
|
|
||||||
if eventuallyMatches {
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, elapsed, int64(0))
|
|
||||||
} else {
|
|
||||||
require.ErrorIs(t, err, os.ErrDeadlineExceeded)
|
|
||||||
assert.Equal(t, int64(-1), elapsed)
|
|
||||||
}
|
|
||||||
assert.Equal(t, 2, conn.reads)
|
|
||||||
assert.Equal(t, 1, conn.deadlineSets)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPLoopback(t *testing.T) {
|
|
||||||
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
|
|
||||||
for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
|
|
||||||
t.Run(network, func(t *testing.T) {
|
|
||||||
conn, err := icmp.ListenPacket(network, family.listenAddr)
|
|
||||||
if err != nil {
|
|
||||||
t.Skipf("ICMP socket unavailable: %v", err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
ip := net.ParseIP("127.0.0.1")
|
|
||||||
if family.isIPv6 {
|
|
||||||
ip = net.ParseIP("::1")
|
|
||||||
}
|
|
||||||
var dst net.Addr = &net.IPAddr{IP: ip}
|
|
||||||
if network == family.dgramNetwork {
|
|
||||||
dst = &net.UDPAddr{IP: ip}
|
|
||||||
}
|
|
||||||
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, elapsed, int64(0))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetectICMPMode(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
family *icmpFamily
|
|
||||||
rawErr error
|
|
||||||
udpErr error
|
|
||||||
want icmpMethod
|
|
||||||
wantNetworks []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "IPv4 prefers raw socket when available",
|
|
||||||
family: &icmpV4,
|
|
||||||
want: icmpRaw,
|
|
||||||
wantNetworks: []string{"ip4:icmp"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv4 uses datagram when raw unavailable",
|
|
||||||
family: &icmpV4,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
want: icmpDatagram,
|
|
||||||
wantNetworks: []string{"ip4:icmp", "udp4"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv4 falls back to exec when both unavailable",
|
|
||||||
family: &icmpV4,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
udpErr: errors.New("protocol not supported"),
|
|
||||||
want: icmpExecFallback,
|
|
||||||
wantNetworks: []string{"ip4:icmp", "udp4"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 prefers raw socket when available",
|
|
||||||
family: &icmpV6,
|
|
||||||
want: icmpRaw,
|
|
||||||
wantNetworks: []string{"ip6:ipv6-icmp"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 uses datagram when raw unavailable",
|
|
||||||
family: &icmpV6,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
want: icmpDatagram,
|
|
||||||
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 falls back to exec when both unavailable",
|
|
||||||
family: &icmpV6,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
udpErr: errors.New("protocol not supported"),
|
|
||||||
want: icmpExecFallback,
|
|
||||||
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
calls := make([]string, 0, 2)
|
|
||||||
listen := func(network, listenAddr string) (icmpPacketConn, error) {
|
|
||||||
require.Equal(t, tt.family.listenAddr, listenAddr)
|
|
||||||
calls = append(calls, network)
|
|
||||||
switch network {
|
|
||||||
case tt.family.rawNetwork:
|
|
||||||
if tt.rawErr != nil {
|
|
||||||
return nil, tt.rawErr
|
|
||||||
}
|
|
||||||
case tt.family.dgramNetwork:
|
|
||||||
if tt.udpErr != nil {
|
|
||||||
return nil, tt.udpErr
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
t.Fatalf("unexpected network %q", network)
|
|
||||||
}
|
|
||||||
return testICMPPacketConn{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
|
|
||||||
assert.Equal(t, tt.wantNetworks, calls)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveICMPTarget(t *testing.T) {
|
|
||||||
t.Run("IPv4 literal", func(t *testing.T) {
|
|
||||||
family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, family)
|
|
||||||
assert.False(t, family.isIPv6)
|
|
||||||
assert.Equal(t, "127.0.0.1", ip.String())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("IPv6 literal", func(t *testing.T) {
|
|
||||||
family, ip, err := resolveICMPTarget(context.Background(), "::1")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, family)
|
|
||||||
assert.True(t, family.isIPv6)
|
|
||||||
assert.Equal(t, "::1", ip.String())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
|
|
||||||
family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, family)
|
|
||||||
assert.False(t, family.isIPv6)
|
|
||||||
assert.Equal(t, "127.0.0.1", ip.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorProbe performs one check. Errors are recorded as loss by the task runner.
|
|
||||||
// Implementations must honor cancellation and bound their execution time.
|
|
||||||
type monitorProbe func(context.Context, monitor.Config) (int64, error)
|
|
||||||
|
|
||||||
func networkMonitorProbe(client *http.Client) monitorProbe {
|
|
||||||
return func(ctx context.Context, config monitor.Config) (int64, error) {
|
|
||||||
switch config.Protocol {
|
|
||||||
case "icmp":
|
|
||||||
return monitorICMP(ctx, config.Target)
|
|
||||||
case "tcp":
|
|
||||||
return monitorTCP(ctx, config.Target, config.Port)
|
|
||||||
case "http":
|
|
||||||
return monitorHTTP(ctx, client, config.Target)
|
|
||||||
case "dns":
|
|
||||||
return monitorDNS(ctx, config.Target)
|
|
||||||
default:
|
|
||||||
return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorTCP measures connection establishment time, including address fallback
|
|
||||||
// but excluding DNS resolution.
|
|
||||||
// Returns -1 and an error on failure.
|
|
||||||
func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
// Resolve DNS first, outside the timing window but within the probe deadline.
|
|
||||||
ips, err := net.DefaultResolver.LookupHost(ctx, target)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
if len(ips) == 0 {
|
|
||||||
return -1, errors.New("no addresses resolved for TCP monitor")
|
|
||||||
}
|
|
||||||
portString := fmt.Sprintf("%d", port)
|
|
||||||
deadline, _ := ctx.Deadline()
|
|
||||||
|
|
||||||
// Share the remaining probe budget across addresses so an unresponsive
|
|
||||||
// first address cannot consume all the time available for alternatives.
|
|
||||||
start := time.Now()
|
|
||||||
for i, ip := range ips {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
|
|
||||||
var conn net.Conn
|
|
||||||
conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
responseUs := time.Since(start).Microseconds()
|
|
||||||
conn.Close()
|
|
||||||
return responseUs, nil
|
|
||||||
}
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.
|
|
||||||
func monitorDNS(ctx context.Context, target string) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
ips, err := net.DefaultResolver.LookupHost(ctx, target)
|
|
||||||
if err != nil || len(ips) == 0 {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
return time.Since(start).Microseconds(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
|
|
||||||
func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
|
|
||||||
if client == nil {
|
|
||||||
client = http.DefaultClient
|
|
||||||
}
|
|
||||||
start := time.Now()
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode >= 400 {
|
|
||||||
return -1, fmt.Errorf("HTTP error: %s", resp.Status)
|
|
||||||
}
|
|
||||||
return time.Since(start).Microseconds(), nil
|
|
||||||
}
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
monitorResumeHeartbeat = 10 * time.Second
|
|
||||||
// Allow scheduling jitter without mistaking an ordinary tick for resume.
|
|
||||||
monitorResumeGap = 2 * monitorResumeHeartbeat
|
|
||||||
monitorResumePause = 10 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorResumeGuard detects likely suspend/resume using wall time. A long
|
|
||||||
// process stall or forward clock adjustment can also trigger the bounded pause.
|
|
||||||
// One heartbeat is shared by all configured monitors.
|
|
||||||
type monitorResumeGuard struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
stop chan struct{}
|
|
||||||
lastTick time.Time
|
|
||||||
pauseUntil time.Time
|
|
||||||
generation uint32
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *monitorResumeGuard) start() {
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
if g.stop != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
stop := make(chan struct{})
|
|
||||||
g.stop = stop
|
|
||||||
g.lastTick = time.Now().Round(0)
|
|
||||||
g.pauseUntil = time.Time{}
|
|
||||||
go func() {
|
|
||||||
ticker := time.NewTicker(monitorResumeHeartbeat)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
g.mu.Lock()
|
|
||||||
if g.stop == stop {
|
|
||||||
g.observe(time.Now())
|
|
||||||
}
|
|
||||||
g.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *monitorResumeGuard) shutdown() {
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
if g.stop != nil {
|
|
||||||
close(g.stop)
|
|
||||||
g.stop = nil
|
|
||||||
g.generation++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// observe requires mu. Strip the monotonic component because it can stop during
|
|
||||||
// suspend. Read the current time rather than the ticker's queued timestamp.
|
|
||||||
func (g *monitorResumeGuard) observe(now time.Time) {
|
|
||||||
now = now.Round(0)
|
|
||||||
if now.Sub(g.lastTick) > monitorResumeGap {
|
|
||||||
g.pauseUntil = now.Add(monitorResumePause)
|
|
||||||
g.generation++
|
|
||||||
}
|
|
||||||
g.lastTick = now
|
|
||||||
}
|
|
||||||
|
|
||||||
// snapshot also observes time so a probe waking before the heartbeat detects
|
|
||||||
// resume itself. A changed generation invalidates probes spanning suspend.
|
|
||||||
func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
|
|
||||||
if g == nil {
|
|
||||||
return 0, true
|
|
||||||
}
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
if g.stop == nil {
|
|
||||||
return g.generation, true
|
|
||||||
}
|
|
||||||
g.observe(time.Now())
|
|
||||||
return g.generation, !g.lastTick.Before(g.pauseUntil)
|
|
||||||
}
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func simulateMonitorSleep(g *monitorResumeGuard) {
|
|
||||||
g.mu.Lock()
|
|
||||||
g.lastTick = time.Now().Add(-time.Hour).Round(0)
|
|
||||||
g.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumePause(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var g monitorResumeGuard
|
|
||||||
g.start()
|
|
||||||
defer g.shutdown()
|
|
||||||
generation, allowed := g.snapshot()
|
|
||||||
require.True(t, allowed)
|
|
||||||
// Heartbeats alone must keep the guard current between infrequent probes.
|
|
||||||
time.Sleep(time.Minute)
|
|
||||||
synctest.Wait()
|
|
||||||
steadyGeneration, allowed := g.snapshot()
|
|
||||||
require.True(t, allowed)
|
|
||||||
require.Equal(t, generation, steadyGeneration)
|
|
||||||
// The probe, rather than the heartbeat, must detect this gap.
|
|
||||||
simulateMonitorSleep(&g)
|
|
||||||
next, allowed := g.snapshot()
|
|
||||||
assert.False(t, allowed)
|
|
||||||
assert.NotEqual(t, generation, next)
|
|
||||||
time.Sleep(9 * time.Second)
|
|
||||||
_, allowed = g.snapshot()
|
|
||||||
assert.False(t, allowed)
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
end, allowed := g.snapshot()
|
|
||||||
assert.True(t, allowed)
|
|
||||||
assert.Equal(t, next, end)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumeGuardLifecycle(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
|
|
||||||
defer pm.Stop()
|
|
||||||
assert.Nil(t, pm.resumeGuard.stop)
|
|
||||||
pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
|
|
||||||
stop := pm.resumeGuard.stop
|
|
||||||
require.NotNil(t, stop)
|
|
||||||
pm.DeleteMonitor("a")
|
|
||||||
assert.Equal(t, stop, pm.resumeGuard.stop)
|
|
||||||
pm.DeleteMonitor("b")
|
|
||||||
assert.Nil(t, pm.resumeGuard.stop)
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
default:
|
|
||||||
t.Fatal("heartbeat was not stopped")
|
|
||||||
}
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
_, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, allowed := pm.resumeGuard.snapshot()
|
|
||||||
assert.True(t, allowed, "idle time must not trigger a resume pause")
|
|
||||||
pm.SyncMonitors(nil)
|
|
||||||
assert.Nil(t, pm.resumeGuard.stop)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var g monitorResumeGuard
|
|
||||||
g.start()
|
|
||||||
defer g.shutdown()
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "test"})
|
|
||||||
defer task.cancel()
|
|
||||||
task.resumeGuard = &g
|
|
||||||
result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
simulateMonitorSleep(&g)
|
|
||||||
return 0, errors.New("network not ready")
|
|
||||||
})
|
|
||||||
assert.Nil(t, result)
|
|
||||||
assert.Empty(t, task.history.samples)
|
|
||||||
// Explicit requests may still run during the pause and record real failures.
|
|
||||||
result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
return 0, errors.New("unreachable")
|
|
||||||
})
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
calls.Add(1)
|
|
||||||
return 1, nil
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
|
|
||||||
simulateMonitorSleep(&pm.resumeGuard)
|
|
||||||
pm.resumeGuard.snapshot()
|
|
||||||
time.Sleep(9 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Zero(t, calls.Load())
|
|
||||||
assert.Empty(t, pm.GetResults(1000))
|
|
||||||
time.Sleep(2 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Positive(t, calls.Load())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,60 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"math/rand"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (pm *MonitorManager) startMonitor(task *monitorTask) {
|
|
||||||
interval := time.Duration(task.config.Interval) * time.Second
|
|
||||||
if interval < time.Second {
|
|
||||||
interval = 30 * time.Second
|
|
||||||
}
|
|
||||||
delay := getStagger(interval.Milliseconds())
|
|
||||||
slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
|
|
||||||
go runMonitorSchedule(task.ctx, interval, delay, func() {
|
|
||||||
if _, allowed := task.resumeGuard.snapshot(); allowed {
|
|
||||||
task.runProbe(pm.probe)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// runMonitorSchedule owns only timing. Checks run serially, and slow checks
|
|
||||||
// naturally drop missed ticks rather than building an execution backlog.
|
|
||||||
func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
|
|
||||||
timer := time.NewTimer(delay)
|
|
||||||
defer timer.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
run()
|
|
||||||
ticker := time.NewTicker(interval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
run()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// getStagger returns an initial delay between half an interval and one interval.
|
|
||||||
func getStagger(intervalMilli int64) time.Duration {
|
|
||||||
delay := rand.Intn(int(intervalMilli))
|
|
||||||
if delay < int(intervalMilli)/2 {
|
|
||||||
delay += int(intervalMilli) / 2
|
|
||||||
}
|
|
||||||
return time.Duration(delay) * time.Millisecond
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorScheduleTiming(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
defer cancel()
|
|
||||||
var calls atomic.Int32
|
|
||||||
go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
|
|
||||||
synctest.Wait()
|
|
||||||
time.Sleep(4 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 0, int(calls.Load()))
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()))
|
|
||||||
time.Sleep(10 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
cancel()
|
|
||||||
synctest.Wait()
|
|
||||||
time.Sleep(time.Minute)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorScheduleSlowProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
defer cancel()
|
|
||||||
var calls atomic.Int32
|
|
||||||
release := make(chan struct{})
|
|
||||||
go runMonitorSchedule(ctx, time.Second, 0, func() {
|
|
||||||
calls.Add(1)
|
|
||||||
select {
|
|
||||||
case <-release:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
})
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()))
|
|
||||||
time.Sleep(time.Minute)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
|
|
||||||
close(release)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
cancel()
|
|
||||||
synctest.Wait()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
release := make(chan struct{})
|
|
||||||
cfg := monitor.Config{ID: "test", Interval: 10}
|
|
||||||
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
|
|
||||||
assert.Equal(t, cfg, config)
|
|
||||||
calls.Add(1)
|
|
||||||
<-release
|
|
||||||
return 42, nil
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
task := newMonitorTask(cfg)
|
|
||||||
pm.monitors[cfg.ID] = task
|
|
||||||
go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
|
|
||||||
synctest.Wait()
|
|
||||||
results := make(chan *monitor.Result, 2)
|
|
||||||
for range 2 {
|
|
||||||
go func() {
|
|
||||||
result, _ := pm.UpsertMonitor(cfg, true)
|
|
||||||
results <- result
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()))
|
|
||||||
assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
|
|
||||||
close(release)
|
|
||||||
synctest.Wait()
|
|
||||||
first, second := <-results, <-results
|
|
||||||
require.NotNil(t, first)
|
|
||||||
require.NotNil(t, second)
|
|
||||||
assert.Equal(t, int64(42), first.AvgResponse)
|
|
||||||
assert.Equal(t, first, second)
|
|
||||||
assert.NotSame(t, first, second, "callers must not share mutable result pointers")
|
|
||||||
assert.Len(t, task.history.samples, 1)
|
|
||||||
// A later explicit request must still perform a fresh probe.
|
|
||||||
_, err := pm.UpsertMonitor(cfg, true)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
assert.Len(t, task.history.samples, 2)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
cfg := monitor.Config{ID: "test", Interval: 10}
|
|
||||||
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
|
|
||||||
if config.Interval == 10 {
|
|
||||||
<-ctx.Done()
|
|
||||||
return 0, ctx.Err()
|
|
||||||
}
|
|
||||||
return 30, nil
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
task := newMonitorTask(cfg)
|
|
||||||
task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
|
|
||||||
pm.monitors[cfg.ID] = task
|
|
||||||
results := make(chan *monitor.Result, 2)
|
|
||||||
for range 2 {
|
|
||||||
go func() {
|
|
||||||
result, _ := pm.UpsertMonitor(cfg, true)
|
|
||||||
results <- result
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
synctest.Wait()
|
|
||||||
updated := cfg
|
|
||||||
updated.Interval = 20
|
|
||||||
result, err := pm.UpsertMonitor(updated, true)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, int64(20), result.AvgResponse)
|
|
||||||
assert.Zero(t, result.PacketLoss)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Nil(t, <-results)
|
|
||||||
assert.Nil(t, <-results)
|
|
||||||
assert.Len(t, task.history.samples, 1)
|
|
||||||
assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
<-ctx.Done()
|
|
||||||
return 0, ctx.Err()
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
start := time.Now()
|
|
||||||
result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, 3*time.Second, time.Since(start))
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
assert.NoError(t, pm.monitors["test"].ctx.Err())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
const monitorFailureLogInterval = 5 * time.Minute
|
|
||||||
|
|
||||||
// monitorTask coordinates a probe and its history for one immutable configuration.
|
|
||||||
type monitorTask struct {
|
|
||||||
config monitor.Config
|
|
||||||
ctx context.Context
|
|
||||||
cancel context.CancelFunc
|
|
||||||
history *monitorHistory
|
|
||||||
resumeGuard *monitorResumeGuard
|
|
||||||
runMu sync.Mutex
|
|
||||||
inflight *monitorRun
|
|
||||||
lastFailureLog int64 // Unix nanoseconds
|
|
||||||
}
|
|
||||||
|
|
||||||
type monitorRun struct {
|
|
||||||
done chan struct{}
|
|
||||||
result *monitor.Result // published by closing done; never mutated afterwards
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorTask(config monitor.Config) *monitorTask {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
|
|
||||||
// Serialize cancellation with publication, so canceled probes cannot enter
|
|
||||||
// history copied into a replacement task.
|
|
||||||
task.cancel = func() {
|
|
||||||
task.runMu.Lock()
|
|
||||||
cancel()
|
|
||||||
task.runMu.Unlock()
|
|
||||||
}
|
|
||||||
return task
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
|
|
||||||
task := newMonitorTask(config)
|
|
||||||
if existing != nil {
|
|
||||||
task.history = existing.history.clone()
|
|
||||||
}
|
|
||||||
return task
|
|
||||||
}
|
|
||||||
|
|
||||||
// runProbe shares an in-flight check between scheduled and immediate requests.
|
|
||||||
// Every completed check contributes exactly one sample, regardless of how many
|
|
||||||
// callers were waiting for it. No task or history lock is held during network I/O.
|
|
||||||
func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
|
|
||||||
task.runMu.Lock()
|
|
||||||
if task.ctx.Err() != nil {
|
|
||||||
task.runMu.Unlock()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if run := task.inflight; run != nil {
|
|
||||||
task.runMu.Unlock()
|
|
||||||
select {
|
|
||||||
case <-task.ctx.Done():
|
|
||||||
return nil
|
|
||||||
case <-run.done:
|
|
||||||
if task.ctx.Err() != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return copyMonitorResult(run.result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
run := &monitorRun{done: make(chan struct{})}
|
|
||||||
task.inflight = run
|
|
||||||
task.runMu.Unlock()
|
|
||||||
|
|
||||||
generation, _ := task.resumeGuard.snapshot()
|
|
||||||
responseUs, err := probe(task.ctx, task.config)
|
|
||||||
var logFailure bool
|
|
||||||
task.runMu.Lock()
|
|
||||||
currentGeneration, _ := task.resumeGuard.snapshot()
|
|
||||||
if task.ctx.Err() == nil && generation == currentGeneration {
|
|
||||||
now := time.Now()
|
|
||||||
if err != nil {
|
|
||||||
responseUs = -1
|
|
||||||
logAt := now.UnixNano()
|
|
||||||
if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
|
|
||||||
logFailure = true
|
|
||||||
task.lastFailureLog = logAt
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
task.lastFailureLog = 0
|
|
||||||
}
|
|
||||||
result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
|
|
||||||
run.result = &result
|
|
||||||
}
|
|
||||||
|
|
||||||
task.inflight = nil
|
|
||||||
close(run.done)
|
|
||||||
task.runMu.Unlock()
|
|
||||||
if logFailure {
|
|
||||||
slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
|
|
||||||
}
|
|
||||||
if task.ctx.Err() != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return copyMonitorResult(run.result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyMonitorResult(result *monitor.Result) *monitor.Result {
|
|
||||||
if result == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
copy := *result
|
|
||||||
return ©
|
|
||||||
}
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"log/slog"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorFailureLogCooldown(t *testing.T) {
|
|
||||||
var logs bytes.Buffer
|
|
||||||
previous := slog.Default()
|
|
||||||
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
|
|
||||||
t.Cleanup(func() { slog.SetDefault(previous) })
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
|
|
||||||
defer task.cancel()
|
|
||||||
failure := errors.New("connection refused")
|
|
||||||
probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
|
|
||||||
var samples int64
|
|
||||||
check := func(wantLog bool) {
|
|
||||||
t.Helper()
|
|
||||||
logs.Reset()
|
|
||||||
result := task.runProbe(probe)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
samples++
|
|
||||||
assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
|
|
||||||
if !wantLog {
|
|
||||||
assert.Empty(t, logs.String())
|
|
||||||
} else {
|
|
||||||
assert.Contains(t, logs.String(), `msg="monitor failed"`)
|
|
||||||
assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
check(true)
|
|
||||||
check(false)
|
|
||||||
time.Sleep(5*time.Minute - time.Nanosecond)
|
|
||||||
check(false)
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
check(true)
|
|
||||||
check(false)
|
|
||||||
time.Sleep(5 * time.Minute)
|
|
||||||
check(true)
|
|
||||||
check(false)
|
|
||||||
|
|
||||||
// Recovery clears the cooldown.
|
|
||||||
failure = nil
|
|
||||||
check(false)
|
|
||||||
failure = errors.New("connection refused again")
|
|
||||||
check(true)
|
|
||||||
|
|
||||||
// Another monitor has its own cooldown.
|
|
||||||
other := newMonitorTask(task.config)
|
|
||||||
defer other.cancel()
|
|
||||||
logs.Reset()
|
|
||||||
require.NotNil(t, other.runProbe(probe))
|
|
||||||
assert.Contains(t, logs.String(), `msg="monitor failed"`)
|
|
||||||
|
|
||||||
// A canceled probe must not publish a failure or emit a warning.
|
|
||||||
logs.Reset()
|
|
||||||
result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
other.cancel()
|
|
||||||
return -1, context.Canceled
|
|
||||||
})
|
|
||||||
assert.Nil(t, result)
|
|
||||||
assert.Empty(t, logs.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,524 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/binary"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"golang.org/x/net/dns/dnsmessage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
|
|
||||||
now := time.Now().UTC()
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
|
|
||||||
|
|
||||||
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
|
|
||||||
result, ok := results["monitor-1"]
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, int64(30), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(25), result.AvgResponse1h)
|
|
||||||
assert.Equal(t, int64(30), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(10), result.MinResponse1h)
|
|
||||||
assert.Equal(t, int64(30), result.MaxResponse)
|
|
||||||
assert.Equal(t, int64(40), result.MaxResponse1h)
|
|
||||||
assert.Equal(t, 50.0, result.PacketLoss)
|
|
||||||
assert.Equal(t, 20.0, result.PacketLoss1h)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
|
|
||||||
now := time.Now().UTC()
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
|
|
||||||
|
|
||||||
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
|
|
||||||
result, ok := results["monitor-1"]
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, int64(0), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(0), result.AvgResponse1h)
|
|
||||||
assert.Equal(t, int64(0), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(0), result.MinResponse1h)
|
|
||||||
assert.Equal(t, int64(0), result.MaxResponse)
|
|
||||||
assert.Equal(t, int64(0), result.MaxResponse1h)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss1h)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
|
|
||||||
cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
|
||||||
assert.Equal(t, "monitor-1", cfg.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
|
|
||||||
validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
_, validExists := pm.monitors[validCfg.ID]
|
|
||||||
_, invalidExists := pm.monitors[invalidCfg.ID]
|
|
||||||
assert.True(t, validExists)
|
|
||||||
assert.False(t, invalidExists)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
|
|
||||||
keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
|
|
||||||
keptTask := newMonitorTask(keepCfg)
|
|
||||||
removedTask := newMonitorTask(removeCfg)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{
|
|
||||||
keepCfg.ID: keptTask,
|
|
||||||
removeCfg.ID: removedTask,
|
|
||||||
}
|
|
||||||
|
|
||||||
pm.SyncMonitors([]monitor.Config{keepCfg})
|
|
||||||
|
|
||||||
assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
|
|
||||||
_, exists := pm.monitors[removeCfg.ID]
|
|
||||||
assert.False(t, exists)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-removedTask.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected removed monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-keptTask.ctx.Done():
|
|
||||||
t.Fatal("expected existing monitor task to remain active")
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
|
|
||||||
originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
|
|
||||||
updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
|
|
||||||
originalTask := newMonitorTask(originalCfg)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{
|
|
||||||
originalCfg.ID: originalTask,
|
|
||||||
}
|
|
||||||
|
|
||||||
pm.SyncMonitors([]monitor.Config{updatedCfg})
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
restartedTask := pm.monitors[updatedCfg.ID]
|
|
||||||
assert.NotSame(t, originalTask, restartedTask)
|
|
||||||
assert.Equal(t, updatedCfg, restartedTask.config)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-originalTask.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected changed monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
pm := &MonitorManager{
|
|
||||||
monitors: make(map[string]*monitorTask),
|
|
||||||
probe: networkMonitorProbe(server.Client()),
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
|
|
||||||
Action: monitor.SyncActionUpsert,
|
|
||||||
Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
|
|
||||||
RunNow: true,
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
|
|
||||||
assert.Equal(t, 0.0, resp.Result.PacketLoss)
|
|
||||||
assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
|
|
||||||
|
|
||||||
task := pm.monitors["monitor-1"]
|
|
||||||
require.NotNil(t, task)
|
|
||||||
task.history.mu.Lock()
|
|
||||||
defer task.history.mu.Unlock()
|
|
||||||
require.Len(t, task.history.samples, 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
|
|
||||||
originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
|
||||||
updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
existingTask := newMonitorTask(originalCfg)
|
|
||||||
existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
|
|
||||||
existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
|
|
||||||
|
|
||||||
result, err := pm.UpsertMonitor(updatedCfg, false)
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Nil(t, result)
|
|
||||||
|
|
||||||
updatedTask := pm.monitors[updatedCfg.ID]
|
|
||||||
require.NotNil(t, updatedTask)
|
|
||||||
assert.NotSame(t, existingTask, updatedTask)
|
|
||||||
assert.Equal(t, updatedCfg, updatedTask.config)
|
|
||||||
|
|
||||||
updatedTask.history.mu.Lock()
|
|
||||||
defer updatedTask.history.mu.Unlock()
|
|
||||||
require.Len(t, updatedTask.history.samples, 1)
|
|
||||||
assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
|
|
||||||
|
|
||||||
agg := updatedTask.history.aggregateLocked(time.Hour, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(2), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(2), agg.successCount)
|
|
||||||
assert.Equal(t, int64(18), agg.avgResponse())
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-existingTask.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected original monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
|
|
||||||
config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
|
||||||
task := newMonitorTask(config)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{config.ID: task}
|
|
||||||
|
|
||||||
_, err := pm.HandleSyncRequest(monitor.SyncRequest{
|
|
||||||
Action: monitor.SyncActionDelete,
|
|
||||||
Config: monitor.Config{ID: config.ID},
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, exists := pm.monitors[config.ID]
|
|
||||||
assert.False(t, exists)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-task.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected deleted monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerGetRandomDelay(t *testing.T) {
|
|
||||||
for i := 1000; i < 360_000; i += 1000 {
|
|
||||||
delay := getStagger(int64(i))
|
|
||||||
assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
|
|
||||||
assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHTTP(t *testing.T) {
|
|
||||||
t.Run("success", func(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("server error", func(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
http.Error(w, "boom", http.StatusInternalServerError)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorTCP(t *testing.T) {
|
|
||||||
t.Run("success", func(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer listener.Close()
|
|
||||||
|
|
||||||
accepted := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(accepted)
|
|
||||||
conn, err := listener.Accept()
|
|
||||||
if err == nil {
|
|
||||||
_ = conn.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
port := uint16(listener.Addr().(*net.TCPAddr).Port)
|
|
||||||
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
<-accepted
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("connection failure", func(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
port := uint16(listener.Addr().(*net.TCPAddr).Port)
|
|
||||||
require.NoError(t, listener.Close())
|
|
||||||
|
|
||||||
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorTCPAddressFallback(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
ips []string
|
|
||||||
loss bool
|
|
||||||
}{
|
|
||||||
{"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
|
|
||||||
{"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
|
|
||||||
{"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer listener.Close()
|
|
||||||
original := net.DefaultResolver
|
|
||||||
net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
|
|
||||||
defer func() { net.DefaultResolver = original }()
|
|
||||||
|
|
||||||
// Verify the resolver preserves the intended order, so success cannot
|
|
||||||
// accidentally bypass the failed first address in the regression case.
|
|
||||||
ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, tc.ips, ips)
|
|
||||||
responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
|
|
||||||
if tc.loss {
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tcpMonitorTestResolver supplies multiple A records without external DNS.
|
|
||||||
func tcpMonitorTestResolver(ips []string) *net.Resolver {
|
|
||||||
return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
client, server := net.Pipe()
|
|
||||||
go func() {
|
|
||||||
defer server.Close()
|
|
||||||
// net.Resolver uses TCP framing when its connection is not a PacketConn.
|
|
||||||
var size uint16
|
|
||||||
if err := binary.Read(server, binary.BigEndian, &size); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
packet := make([]byte, size)
|
|
||||||
if _, err := io.ReadFull(server, packet); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var msg dnsmessage.Message
|
|
||||||
if err := msg.Unpack(packet); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
msg.Header.Response = true
|
|
||||||
msg.Header.RecursionAvailable = true
|
|
||||||
for _, question := range msg.Questions {
|
|
||||||
if question.Type != dnsmessage.TypeA {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, ip := range ips {
|
|
||||||
msg.Answers = append(msg.Answers, dnsmessage.Resource{
|
|
||||||
Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
|
|
||||||
Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
packet, err := msg.Pack()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
|
|
||||||
_, _ = server.Write(append(response, packet...))
|
|
||||||
}()
|
|
||||||
return client, nil
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorDNS(t *testing.T) {
|
|
||||||
t.Run("success", func(t *testing.T) {
|
|
||||||
responseUs, err := monitorDNS(context.Background(), "localhost")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("lookup failure", func(t *testing.T) {
|
|
||||||
responseUs, err := monitorDNS(context.Background(), "")
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
|
|
||||||
for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
|
|
||||||
t.Run(action, func(t *testing.T) {
|
|
||||||
started := make(chan struct{})
|
|
||||||
canceled := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
close(started)
|
|
||||||
select {
|
|
||||||
case <-r.Context().Done():
|
|
||||||
close(canceled)
|
|
||||||
case <-release:
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
defer close(release)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
defer pm.Stop()
|
|
||||||
cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
|
|
||||||
task := newMonitorTask(cfg)
|
|
||||||
// Seed history to ensure a canceled RunNow does not return an old result.
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
|
|
||||||
pm.monitors[cfg.ID] = task
|
|
||||||
done := make(chan *monitor.Result, 1)
|
|
||||||
go func() {
|
|
||||||
result, _ := pm.UpsertMonitor(cfg, true)
|
|
||||||
done <- result
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("probe did not start")
|
|
||||||
}
|
|
||||||
updated := cfg
|
|
||||||
updated.Interval--
|
|
||||||
switch action {
|
|
||||||
case "stop":
|
|
||||||
pm.Stop()
|
|
||||||
case "delete":
|
|
||||||
pm.DeleteMonitor(cfg.ID)
|
|
||||||
case "upsert":
|
|
||||||
_, err := pm.UpsertMonitor(updated, false)
|
|
||||||
require.NoError(t, err)
|
|
||||||
case "sync replace":
|
|
||||||
pm.SyncMonitors([]monitor.Config{updated})
|
|
||||||
case "sync remove":
|
|
||||||
pm.SyncMonitors(nil)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-canceled:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("active HTTP request was not canceled")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case result := <-done:
|
|
||||||
assert.Nil(t, result)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("RunNow did not return after cancellation")
|
|
||||||
}
|
|
||||||
task.history.mu.Lock()
|
|
||||||
assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
|
|
||||||
task.history.mu.Unlock()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResolutionCancellation(t *testing.T) {
|
|
||||||
for _, protocol := range []string{"tcp", "dns", "icmp"} {
|
|
||||||
t.Run(protocol, func(t *testing.T) {
|
|
||||||
started := make(chan struct{}, 1)
|
|
||||||
original := net.DefaultResolver
|
|
||||||
net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
select {
|
|
||||||
case started <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
<-ctx.Done()
|
|
||||||
return nil, ctx.Err()
|
|
||||||
}}
|
|
||||||
defer func() { net.DefaultResolver = original }()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
var err error
|
|
||||||
switch protocol {
|
|
||||||
case "tcp":
|
|
||||||
_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
|
|
||||||
case "dns":
|
|
||||||
_, err = monitorDNS(ctx, "monitor-cancellation.invalid.")
|
|
||||||
case "icmp":
|
|
||||||
_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
|
|
||||||
}
|
|
||||||
done <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("lookup did not start")
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.Error(t, err)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("lookup did not cancel")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
select {
|
|
||||||
case <-r.Context().Done():
|
|
||||||
case <-release:
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
defer close(release)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
|
|
||||||
defer task.cancel()
|
|
||||||
|
|
||||||
result := task.runProbe(pm.probe)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss1h)
|
|
||||||
require.Len(t, task.history.samples, 1)
|
|
||||||
assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
|
|
||||||
assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
|
|
||||||
}
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
//go:build !windows && !freebsd
|
//go:build !windows
|
||||||
|
|
||||||
package agent
|
package agent
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
//go:build freebsd
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"github.com/shirou/gopsutil/v4/sensors"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
var getSensorTemps = func(ctx context.Context) ([]sensors.TemperatureStat, error) {
|
|
||||||
return getFreeBSDSensorTemps(ctx, unix.SysctlUint32)
|
|
||||||
}
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
//go:build freebsd || testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/shirou/gopsutil/v4/sensors"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
freebsdZeroCelsiusDeciKelvin = 2731
|
|
||||||
freebsdAcpiThermalZoneCount = 16
|
|
||||||
)
|
|
||||||
|
|
||||||
type freebsdSysctlUintReader func(name string) (uint32, error)
|
|
||||||
|
|
||||||
func getFreeBSDSensorTemps(ctx context.Context, readSysctl freebsdSysctlUintReader) ([]sensors.TemperatureStat, error) {
|
|
||||||
cpuCount, err := readSysctl("hw.ncpu")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
temps := make([]sensors.TemperatureStat, 0, int(cpuCount)+freebsdAcpiThermalZoneCount)
|
|
||||||
for cpu := range cpuCount {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return temps, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
sysctlName := fmt.Sprintf("dev.cpu.%d.temperature", cpu)
|
|
||||||
value, err := readSysctl(sysctlName)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
temp, ok := freebsdDeciKelvinToCelsius(value)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
temps = append(temps, sensors.TemperatureStat{
|
|
||||||
SensorKey: fmt.Sprintf("cpu.%d", cpu),
|
|
||||||
Temperature: temp,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
for zone := 0; zone < freebsdAcpiThermalZoneCount; zone++ {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return temps, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
sysctlName := fmt.Sprintf("hw.acpi.thermal.tz%d.temperature", zone)
|
|
||||||
value, err := readSysctl(sysctlName)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
temp, ok := freebsdDeciKelvinToCelsius(value)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
temps = append(temps, sensors.TemperatureStat{
|
|
||||||
SensorKey: fmt.Sprintf("acpi.thermal.tz%d", zone),
|
|
||||||
Temperature: temp,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return temps, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func freebsdDeciKelvinToCelsius(value uint32) (float64, bool) {
|
|
||||||
if value <= freebsdZeroCelsiusDeciKelvin {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
temp := float64(int64(value)-freebsdZeroCelsiusDeciKelvin) / 10
|
|
||||||
if temp <= 0 || temp >= 200 {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
return temp, true
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
var errFakeFreeBSDSysctlNotFound = errors.New("sysctl not found")
|
|
||||||
|
|
||||||
type fakeFreeBSDSysctls struct {
|
|
||||||
values map[string]uint32
|
|
||||||
errs map[string]error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f fakeFreeBSDSysctls) read(name string) (uint32, error) {
|
|
||||||
if err, ok := f.errs[name]; ok {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if value, ok := f.values[name]; ok {
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
return 0, errFakeFreeBSDSysctlNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFreeBSDDeciKelvinToCelsius(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
value uint32
|
|
||||||
expected float64
|
|
||||||
ok bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "45 Celsius",
|
|
||||||
value: 3181,
|
|
||||||
expected: 45,
|
|
||||||
ok: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fractional Celsius",
|
|
||||||
value: 3186,
|
|
||||||
expected: 45.5,
|
|
||||||
ok: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "zero deci-Kelvin",
|
|
||||||
value: 0,
|
|
||||||
ok: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "zero Celsius",
|
|
||||||
value: freebsdZeroCelsiusDeciKelvin,
|
|
||||||
ok: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "below zero Celsius",
|
|
||||||
value: freebsdZeroCelsiusDeciKelvin - 1,
|
|
||||||
ok: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid signed integer",
|
|
||||||
value: 1<<32 - 1,
|
|
||||||
ok: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unreasonably high Celsius",
|
|
||||||
value: freebsdZeroCelsiusDeciKelvin + 2000,
|
|
||||||
ok: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result, ok := freebsdDeciKelvinToCelsius(tt.value)
|
|
||||||
assert.Equal(t, tt.ok, ok)
|
|
||||||
assert.InDelta(t, tt.expected, result, 0.001)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetFreeBSDSensorTemps(t *testing.T) {
|
|
||||||
reader := fakeFreeBSDSysctls{
|
|
||||||
values: map[string]uint32{
|
|
||||||
"hw.ncpu": 4,
|
|
||||||
"dev.cpu.0.temperature": 3231,
|
|
||||||
"dev.cpu.1.temperature": 3242,
|
|
||||||
"dev.cpu.3.temperature": freebsdZeroCelsiusDeciKelvin,
|
|
||||||
"hw.acpi.thermal.tz0.temperature": 3101,
|
|
||||||
"hw.acpi.thermal.tz2.temperature": 3116,
|
|
||||||
"hw.acpi.thermal.tz3.temperature": freebsdZeroCelsiusDeciKelvin,
|
|
||||||
"unrelated.sensor.value": 9999,
|
|
||||||
"dev.cpu.99.temperature": 9999,
|
|
||||||
"dev.amdtemp.0.core0.foo": 9999,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
temps, err := getFreeBSDSensorTemps(context.Background(), reader.read)
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, temps, 4)
|
|
||||||
assert.Equal(t, "cpu.0", temps[0].SensorKey)
|
|
||||||
assert.InDelta(t, 50.0, temps[0].Temperature, 0.001)
|
|
||||||
assert.Equal(t, "cpu.1", temps[1].SensorKey)
|
|
||||||
assert.InDelta(t, 51.1, temps[1].Temperature, 0.001)
|
|
||||||
assert.Equal(t, "acpi.thermal.tz0", temps[2].SensorKey)
|
|
||||||
assert.InDelta(t, 37.0, temps[2].Temperature, 0.001)
|
|
||||||
assert.Equal(t, "acpi.thermal.tz2", temps[3].SensorKey)
|
|
||||||
assert.InDelta(t, 38.5, temps[3].Temperature, 0.001)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetFreeBSDSensorTempsCpuCountError(t *testing.T) {
|
|
||||||
reader := fakeFreeBSDSysctls{
|
|
||||||
errs: map[string]error{
|
|
||||||
"hw.ncpu": errors.New("permission denied"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
temps, err := getFreeBSDSensorTemps(context.Background(), reader.read)
|
|
||||||
|
|
||||||
assert.Nil(t, temps)
|
|
||||||
assert.EqualError(t, err, "permission denied")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetFreeBSDSensorTempsNoTemperatureSysctls(t *testing.T) {
|
|
||||||
reader := fakeFreeBSDSysctls{
|
|
||||||
values: map[string]uint32{"hw.ncpu": 2},
|
|
||||||
}
|
|
||||||
|
|
||||||
temps, err := getFreeBSDSensorTemps(context.Background(), reader.read)
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, temps)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetFreeBSDSensorTempsAcpiOnly(t *testing.T) {
|
|
||||||
reader := fakeFreeBSDSysctls{
|
|
||||||
values: map[string]uint32{
|
|
||||||
"hw.ncpu": 0,
|
|
||||||
"hw.acpi.thermal.tz0.temperature": 3081,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
temps, err := getFreeBSDSensorTemps(context.Background(), reader.read)
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, temps, 1)
|
|
||||||
assert.Equal(t, "acpi.thermal.tz0", temps[0].SensorKey)
|
|
||||||
assert.InDelta(t, 35.0, temps[0].Temperature, 0.001)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetFreeBSDSensorTempsContextCancelled(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
reader := fakeFreeBSDSysctls{
|
|
||||||
values: map[string]uint32{"hw.ncpu": 2},
|
|
||||||
}
|
|
||||||
|
|
||||||
temps, err := getFreeBSDSensorTemps(ctx, reader.read)
|
|
||||||
|
|
||||||
assert.Empty(t, temps)
|
|
||||||
assert.ErrorIs(t, err, context.Canceled)
|
|
||||||
}
|
|
||||||
@@ -602,9 +602,8 @@ func TestUpdateTemperaturesSkipsOnTimeout(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
originalGetSensorTemps := getSensorTemps
|
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
getSensorTemps = originalGetSensorTemps
|
getSensorTemps = sensors.TemperaturesWithContext
|
||||||
})
|
})
|
||||||
getSensorTemps = func(ctx context.Context) ([]sensors.TemperatureStat, error) {
|
getSensorTemps = func(ctx context.Context) ([]sensors.TemperatureStat, error) {
|
||||||
time.Sleep(50 * time.Millisecond)
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|||||||
@@ -214,12 +214,9 @@ func (lhm *lhmProcess) getTemps(ctx context.Context) (temps []sensors.Temperatur
|
|||||||
return temps, nil
|
return temps, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getSensorTemps is a variable so tests can replace the platform sensor collector.
|
// getSensorTemps attempts to pull sensor temperatures from the embedded LHM process.
|
||||||
var getSensorTemps = getWindowsSensorTemps
|
|
||||||
|
|
||||||
// getWindowsSensorTemps attempts to pull sensor temperatures from the embedded LHM process.
|
|
||||||
// NB: LibreHardwareMonitorLib requires admin privileges to access all available sensors.
|
// NB: LibreHardwareMonitorLib requires admin privileges to access all available sensors.
|
||||||
func getWindowsSensorTemps(ctx context.Context) (temps []sensors.TemperatureStat, err error) {
|
func getSensorTemps(ctx context.Context) (temps []sensors.TemperatureStat, err error) {
|
||||||
defer func() {
|
defer func() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Debug("Error reading sensors", "err", err)
|
slog.Debug("Error reading sensors", "err", err)
|
||||||
|
|||||||
@@ -265,5 +265,6 @@ func (a *Agent) StopServer() error {
|
|||||||
slog.Info("Stopping SSH server")
|
slog.Info("Stopping SSH server")
|
||||||
_ = a.server.Close()
|
_ = a.server.Close()
|
||||||
a.server = nil
|
a.server = nil
|
||||||
|
a.connectionManager.eventChan <- SSHDisconnect
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -198,28 +198,6 @@ func TestStartServerDisableSSH(t *testing.T) {
|
|||||||
assert.Contains(t, err.Error(), "SSH disabled")
|
assert.Contains(t, err.Error(), "SSH disabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopServerDoesNotBlockWhenEventQueueFull(t *testing.T) {
|
|
||||||
agent := createTestAgent(t)
|
|
||||||
agent.server = &ssh.Server{}
|
|
||||||
agent.connectionManager.eventChan = make(chan ConnectionEvent, 1)
|
|
||||||
agent.connectionManager.eventChan <- WebSocketConnect
|
|
||||||
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
done <- agent.StopServer()
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.NoError(t, err)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("StopServer blocked on the connection event queue")
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Nil(t, agent.server)
|
|
||||||
assert.Equal(t, WebSocketConnect, <-agent.connectionManager.eventChan)
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////
|
||||||
//////////////////// ParseKeys Tests ////////////////////////////
|
//////////////////// ParseKeys Tests ////////////////////////////
|
||||||
/////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////
|
||||||
|
|||||||
+15
-46
@@ -375,15 +375,9 @@ func (sm *SmartManager) parseSmartOutput(deviceInfo *DeviceInfo, output []byte)
|
|||||||
Type string
|
Type string
|
||||||
Parse func([]byte) (bool, int)
|
Parse func([]byte) (bool, int)
|
||||||
}{
|
}{
|
||||||
{Type: "nvme", Parse: func(output []byte) (bool, int) {
|
{Type: "nvme", Parse: sm.parseSmartForNvme},
|
||||||
return sm.parseSmartForNvme(output, deviceInfo.Type)
|
{Type: "sat", Parse: sm.parseSmartForSata},
|
||||||
}},
|
{Type: "scsi", Parse: sm.parseSmartForScsi},
|
||||||
{Type: "sat", Parse: func(output []byte) (bool, int) {
|
|
||||||
return sm.parseSmartForSata(output, deviceInfo.Type)
|
|
||||||
}},
|
|
||||||
{Type: "scsi", Parse: func(output []byte) (bool, int) {
|
|
||||||
return sm.parseSmartForScsi(output, deviceInfo.Type)
|
|
||||||
}},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
deviceType := normalizeParserType(deviceInfo.parserType)
|
deviceType := normalizeParserType(deviceInfo.parserType)
|
||||||
@@ -492,11 +486,10 @@ func (sm *SmartManager) CollectSmart(deviceInfo *DeviceInfo) error {
|
|||||||
return errNoValidSmartData
|
return errNoValidSmartData
|
||||||
}
|
}
|
||||||
|
|
||||||
// slog.Info("collecting SMART data", "device", deviceInfo.Name, "type", deviceInfo.Type, "has_existing_data", sm.hasDataForDevice(deviceInfo))
|
// slog.Info("collecting SMART data", "device", deviceInfo.Name, "type", deviceInfo.Type, "has_existing_data", sm.hasDataForDevice(deviceInfo.Name))
|
||||||
|
|
||||||
// Check if we have existing data for this exact device identity. Multiple
|
// Check if we have any existing data for this device
|
||||||
// bridge slots can share a path, so a name-only match is not sufficient.
|
hasExistingData := sm.hasDataForDevice(deviceInfo.Name)
|
||||||
hasExistingData := sm.hasDataForDevice(deviceInfo)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -599,18 +592,14 @@ func (sm *SmartManager) smartctlArgs(deviceInfo *DeviceInfo, includeStandby bool
|
|||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
|
||||||
// hasDataForDevice checks if we have cached SMART data for a specific device identity.
|
// hasDataForDevice checks if we have cached SMART data for a specific device
|
||||||
func (sm *SmartManager) hasDataForDevice(deviceInfo *DeviceInfo) bool {
|
func (sm *SmartManager) hasDataForDevice(deviceName string) bool {
|
||||||
if deviceInfo == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
sm.Lock()
|
sm.Lock()
|
||||||
defer sm.Unlock()
|
defer sm.Unlock()
|
||||||
|
|
||||||
deviceKey := makeDeviceKey(deviceInfo.Name, deviceInfo.Type)
|
// Check if any cached data has this device name
|
||||||
for _, data := range sm.SmartDataMap {
|
for _, data := range sm.SmartDataMap {
|
||||||
if data != nil && makeDeviceKey(data.DiskName, data.DiskType) == deviceKey {
|
if data != nil && data.DiskName == deviceName {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -758,14 +747,7 @@ func mergeDeviceLists(existing, scanned, configured []*DeviceInfo) []*DeviceInfo
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if existingDev := deviceIndexByName[configuredDevice.Name]; existingDev != nil {
|
if existingDev := deviceIndexByName[configuredDevice.Name]; existingDev != nil {
|
||||||
oldKey := makeDeviceKey(existingDev.Name, existingDev.Type)
|
|
||||||
if prev := existingIndex[key]; prev != nil {
|
|
||||||
preserveVerifiedType(existingDev, prev)
|
|
||||||
}
|
|
||||||
applyConfiguredMetadata(existingDev, configuredDevice)
|
applyConfiguredMetadata(existingDev, configuredDevice)
|
||||||
delete(deviceIndex, oldKey)
|
|
||||||
deviceIndex[makeDeviceKey(existingDev.Name, existingDev.Type)] = existingDev
|
|
||||||
delete(deviceIndexByName, configuredDevice.Name)
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -869,11 +851,9 @@ func (sm *SmartManager) isVirtualDeviceFromStrings(fields ...string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseSmartForSata parses the output of smartctl --all -j for SATA/ATA devices and updates the SmartDataMap.
|
// parseSmartForSata parses the output of smartctl --all -j for SATA/ATA devices and updates the SmartDataMap
|
||||||
// deviceType is the exact type used to identify and query the device; when set,
|
|
||||||
// it takes precedence over the generic type reported by smartctl.
|
|
||||||
// Returns hasValidData and exitStatus
|
// Returns hasValidData and exitStatus
|
||||||
func (sm *SmartManager) parseSmartForSata(output []byte, deviceType string) (bool, int) {
|
func (sm *SmartManager) parseSmartForSata(output []byte) (bool, int) {
|
||||||
var data smart.SmartInfoForSata
|
var data smart.SmartInfoForSata
|
||||||
|
|
||||||
if err := json.Unmarshal(output, &data); err != nil {
|
if err := json.Unmarshal(output, &data); err != nil {
|
||||||
@@ -912,9 +892,6 @@ func (sm *SmartManager) parseSmartForSata(output []byte, deviceType string) (boo
|
|||||||
smartData.SmartStatus = getSmartStatus(smartData.Temperature, data.SmartStatus.Passed)
|
smartData.SmartStatus = getSmartStatus(smartData.Temperature, data.SmartStatus.Passed)
|
||||||
smartData.DiskName = data.Device.Name
|
smartData.DiskName = data.Device.Name
|
||||||
smartData.DiskType = data.Device.Type
|
smartData.DiskType = data.Device.Type
|
||||||
if deviceType != "" {
|
|
||||||
smartData.DiskType = deviceType
|
|
||||||
}
|
|
||||||
|
|
||||||
// get values from ata_device_statistics if necessary
|
// get values from ata_device_statistics if necessary
|
||||||
var ataDeviceStats smart.AtaDeviceStatistics
|
var ataDeviceStats smart.AtaDeviceStatistics
|
||||||
@@ -988,7 +965,7 @@ func findAtaDeviceStatisticsValue(data *smart.SmartInfoForSata, ataDeviceStats *
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sm *SmartManager) parseSmartForScsi(output []byte, deviceType string) (bool, int) {
|
func (sm *SmartManager) parseSmartForScsi(output []byte) (bool, int) {
|
||||||
var data smart.SmartInfoForScsi
|
var data smart.SmartInfoForScsi
|
||||||
|
|
||||||
if err := json.Unmarshal(output, &data); err != nil {
|
if err := json.Unmarshal(output, &data); err != nil {
|
||||||
@@ -1023,9 +1000,6 @@ func (sm *SmartManager) parseSmartForScsi(output []byte, deviceType string) (boo
|
|||||||
smartData.SmartStatus = getSmartStatus(smartData.Temperature, data.SmartStatus.Passed)
|
smartData.SmartStatus = getSmartStatus(smartData.Temperature, data.SmartStatus.Passed)
|
||||||
smartData.DiskName = data.Device.Name
|
smartData.DiskName = data.Device.Name
|
||||||
smartData.DiskType = data.Device.Type
|
smartData.DiskType = data.Device.Type
|
||||||
if deviceType != "" {
|
|
||||||
smartData.DiskType = deviceType
|
|
||||||
}
|
|
||||||
|
|
||||||
attributes := make([]*smart.SmartAttribute, 0, 10)
|
attributes := make([]*smart.SmartAttribute, 0, 10)
|
||||||
attributes = append(attributes, &smart.SmartAttribute{Name: "PowerOnHours", RawValue: data.PowerOnTime.Hours})
|
attributes = append(attributes, &smart.SmartAttribute{Name: "PowerOnHours", RawValue: data.PowerOnTime.Hours})
|
||||||
@@ -1123,11 +1097,9 @@ func (sm *SmartManager) lookupDarwinNvmeCapacity(serial string) uint64 {
|
|||||||
return sm.darwinNvmeCapacity[serial]
|
return sm.darwinNvmeCapacity[serial]
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseSmartForNvme parses the output of smartctl --all -j /dev/nvmeX and updates the SmartDataMap.
|
// parseSmartForNvme parses the output of smartctl --all -j /dev/nvmeX and updates the SmartDataMap
|
||||||
// deviceType is the exact type used to identify and query the device; when set,
|
|
||||||
// it takes precedence over the generic type reported by smartctl.
|
|
||||||
// Returns hasValidData and exitStatus
|
// Returns hasValidData and exitStatus
|
||||||
func (sm *SmartManager) parseSmartForNvme(output []byte, deviceType string) (bool, int) {
|
func (sm *SmartManager) parseSmartForNvme(output []byte) (bool, int) {
|
||||||
data := &smart.SmartInfoForNvme{}
|
data := &smart.SmartInfoForNvme{}
|
||||||
|
|
||||||
if err := json.Unmarshal(output, &data); err != nil {
|
if err := json.Unmarshal(output, &data); err != nil {
|
||||||
@@ -1171,9 +1143,6 @@ func (sm *SmartManager) parseSmartForNvme(output []byte, deviceType string) (boo
|
|||||||
smartData.SmartStatus = getSmartStatus(smartData.Temperature, data.SmartStatus.Passed)
|
smartData.SmartStatus = getSmartStatus(smartData.Temperature, data.SmartStatus.Passed)
|
||||||
smartData.DiskName = data.Device.Name
|
smartData.DiskName = data.Device.Name
|
||||||
smartData.DiskType = data.Device.Type
|
smartData.DiskType = data.Device.Type
|
||||||
if deviceType != "" {
|
|
||||||
smartData.DiskType = deviceType
|
|
||||||
}
|
|
||||||
|
|
||||||
// nvme attributes does not follow the same format as ata attributes,
|
// nvme attributes does not follow the same format as ata attributes,
|
||||||
// so we manually map each field to SmartAttributes
|
// so we manually map each field to SmartAttributes
|
||||||
|
|||||||
+12
-122
@@ -4,7 +4,6 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -25,7 +24,7 @@ func TestParseSmartForScsi(t *testing.T) {
|
|||||||
SmartDataMap: make(map[string]*smart.SmartData),
|
SmartDataMap: make(map[string]*smart.SmartData),
|
||||||
}
|
}
|
||||||
|
|
||||||
hasData, exitStatus := sm.parseSmartForScsi(data, "")
|
hasData, exitStatus := sm.parseSmartForScsi(data)
|
||||||
if !hasData {
|
if !hasData {
|
||||||
t.Fatalf("expected SCSI data to parse successfully")
|
t.Fatalf("expected SCSI data to parse successfully")
|
||||||
}
|
}
|
||||||
@@ -70,7 +69,7 @@ func TestParseSmartForSata(t *testing.T) {
|
|||||||
SmartDataMap: make(map[string]*smart.SmartData),
|
SmartDataMap: make(map[string]*smart.SmartData),
|
||||||
}
|
}
|
||||||
|
|
||||||
hasData, exitStatus := sm.parseSmartForSata(data, "")
|
hasData, exitStatus := sm.parseSmartForSata(data)
|
||||||
require.True(t, hasData)
|
require.True(t, hasData)
|
||||||
assert.Equal(t, 64, exitStatus)
|
assert.Equal(t, 64, exitStatus)
|
||||||
|
|
||||||
@@ -89,31 +88,6 @@ func TestParseSmartForSata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseSmartForSataPreservesFailedAndUnknownStatus(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
temperature int
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "failed", temperature: 30, want: "FAILED"},
|
|
||||||
{name: "unknown", want: "UNKNOWN"},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
jsonPayload := []byte(fmt.Sprintf(`{
|
|
||||||
"device": {"name": "/dev/sda", "type": "sat"},
|
|
||||||
"serial_number": "PRESERVE%s",
|
|
||||||
"temperature": {"current": %d},
|
|
||||||
"ata_smart_attributes": {"table": [{"id": 197, "raw": {"value": 1, "string": "1"}}]}
|
|
||||||
}`, test.name, test.temperature))
|
|
||||||
|
|
||||||
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
|
||||||
hasData, _ := sm.parseSmartForSata(jsonPayload, "")
|
|
||||||
require.True(t, hasData)
|
|
||||||
assert.Equal(t, test.want, sm.SmartDataMap["PRESERVE"+test.name].SmartStatus)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseSmartForSataDeviceStatisticsTemperature(t *testing.T) {
|
func TestParseSmartForSataDeviceStatisticsTemperature(t *testing.T) {
|
||||||
jsonPayload := []byte(`{
|
jsonPayload := []byte(`{
|
||||||
"smartctl": {"exit_status": 0},
|
"smartctl": {"exit_status": 0},
|
||||||
@@ -138,7 +112,7 @@ func TestParseSmartForSataDeviceStatisticsTemperature(t *testing.T) {
|
|||||||
}`)
|
}`)
|
||||||
|
|
||||||
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
||||||
hasData, exitStatus := sm.parseSmartForSata(jsonPayload, "")
|
hasData, exitStatus := sm.parseSmartForSata(jsonPayload)
|
||||||
require.True(t, hasData)
|
require.True(t, hasData)
|
||||||
assert.Equal(t, 0, exitStatus)
|
assert.Equal(t, 0, exitStatus)
|
||||||
|
|
||||||
@@ -173,7 +147,7 @@ func TestParseSmartForSataAtaDeviceStatistics(t *testing.T) {
|
|||||||
}`)
|
}`)
|
||||||
|
|
||||||
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
||||||
hasData, exitStatus := sm.parseSmartForSata(jsonPayload, "")
|
hasData, exitStatus := sm.parseSmartForSata(jsonPayload)
|
||||||
require.True(t, hasData)
|
require.True(t, hasData)
|
||||||
assert.Equal(t, 0, exitStatus)
|
assert.Equal(t, 0, exitStatus)
|
||||||
|
|
||||||
@@ -210,7 +184,7 @@ func TestParseSmartForSataNegativeDeviceStatistics(t *testing.T) {
|
|||||||
}`)
|
}`)
|
||||||
|
|
||||||
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
||||||
hasData, exitStatus := sm.parseSmartForSata(jsonPayload, "")
|
hasData, exitStatus := sm.parseSmartForSata(jsonPayload)
|
||||||
require.True(t, hasData)
|
require.True(t, hasData)
|
||||||
assert.Equal(t, 0, exitStatus)
|
assert.Equal(t, 0, exitStatus)
|
||||||
|
|
||||||
@@ -249,7 +223,7 @@ func TestParseSmartForSataParentheticalRawValue(t *testing.T) {
|
|||||||
|
|
||||||
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
||||||
|
|
||||||
hasData, exitStatus := sm.parseSmartForSata(jsonPayload, "")
|
hasData, exitStatus := sm.parseSmartForSata(jsonPayload)
|
||||||
require.True(t, hasData)
|
require.True(t, hasData)
|
||||||
assert.Equal(t, 0, exitStatus)
|
assert.Equal(t, 0, exitStatus)
|
||||||
|
|
||||||
@@ -271,7 +245,7 @@ func TestParseSmartForNvme(t *testing.T) {
|
|||||||
SmartDataMap: make(map[string]*smart.SmartData),
|
SmartDataMap: make(map[string]*smart.SmartData),
|
||||||
}
|
}
|
||||||
|
|
||||||
hasData, exitStatus := sm.parseSmartForNvme(data, "")
|
hasData, exitStatus := sm.parseSmartForNvme(data)
|
||||||
require.True(t, hasData)
|
require.True(t, hasData)
|
||||||
assert.Equal(t, 0, exitStatus)
|
assert.Equal(t, 0, exitStatus)
|
||||||
|
|
||||||
@@ -294,15 +268,13 @@ func TestParseSmartForNvme(t *testing.T) {
|
|||||||
func TestHasDataForDevice(t *testing.T) {
|
func TestHasDataForDevice(t *testing.T) {
|
||||||
sm := &SmartManager{
|
sm := &SmartManager{
|
||||||
SmartDataMap: map[string]*smart.SmartData{
|
SmartDataMap: map[string]*smart.SmartData{
|
||||||
"serial-1": {DiskName: "/dev/sda", DiskType: "jms56x,0"},
|
"serial-1": {DiskName: "/dev/sda"},
|
||||||
"serial-2": nil,
|
"serial-2": nil,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.True(t, sm.hasDataForDevice(&DeviceInfo{Name: "/dev/sda", Type: "jms56x,0"}))
|
assert.True(t, sm.hasDataForDevice("/dev/sda"))
|
||||||
assert.False(t, sm.hasDataForDevice(&DeviceInfo{Name: "/dev/sda", Type: "jms56x,1"}))
|
assert.False(t, sm.hasDataForDevice("/dev/sdb"))
|
||||||
assert.False(t, sm.hasDataForDevice(&DeviceInfo{Name: "/dev/sdb", Type: "jms56x,0"}))
|
|
||||||
assert.False(t, sm.hasDataForDevice(nil))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDevicesSnapshotReturnsCopy(t *testing.T) {
|
func TestDevicesSnapshotReturnsCopy(t *testing.T) {
|
||||||
@@ -637,74 +609,6 @@ func TestMergeDeviceListsPrefersConfigured(t *testing.T) {
|
|||||||
assert.Equal(t, "sat", byName["/dev/sdb"].Type)
|
assert.Equal(t, "sat", byName["/dev/sdb"].Type)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeDeviceListsExpandsConfiguredDevicesWithSamePath(t *testing.T) {
|
|
||||||
scanned := []*DeviceInfo{
|
|
||||||
{Name: "/dev/sdb", Type: "sat", InfoName: "scan-info", Protocol: "ATA"},
|
|
||||||
}
|
|
||||||
configured := []*DeviceInfo{
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,0", explicitType: true},
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,1", explicitType: true},
|
|
||||||
}
|
|
||||||
|
|
||||||
merged := mergeDeviceLists(nil, scanned, configured)
|
|
||||||
require.Len(t, merged, 2)
|
|
||||||
|
|
||||||
byKey := make(map[deviceKey]*DeviceInfo, len(merged))
|
|
||||||
for _, device := range merged {
|
|
||||||
byKey[makeDeviceKey(device.Name, device.Type)] = device
|
|
||||||
}
|
|
||||||
|
|
||||||
first := byKey[makeDeviceKey("/dev/sdb", "jms56x,0")]
|
|
||||||
require.NotNil(t, first)
|
|
||||||
assert.Equal(t, "scan-info", first.InfoName)
|
|
||||||
assert.Equal(t, "ATA", first.Protocol)
|
|
||||||
assert.True(t, first.explicitType)
|
|
||||||
|
|
||||||
second := byKey[makeDeviceKey("/dev/sdb", "jms56x,1")]
|
|
||||||
require.NotNil(t, second)
|
|
||||||
assert.True(t, second.explicitType)
|
|
||||||
assert.NotContains(t, byKey, makeDeviceKey("/dev/sdb", "sat"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMergeDeviceListsPreservesSamePathVerificationAcrossRescan(t *testing.T) {
|
|
||||||
existing := []*DeviceInfo{
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,0", parserType: "sat", typeVerified: true, explicitType: true},
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,1", parserType: "sat", typeVerified: true, explicitType: true},
|
|
||||||
}
|
|
||||||
scanned := []*DeviceInfo{
|
|
||||||
{Name: "/dev/sdb", Type: "sat", Protocol: "ATA"},
|
|
||||||
}
|
|
||||||
configured := []*DeviceInfo{
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,0", explicitType: true},
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,1", explicitType: true},
|
|
||||||
}
|
|
||||||
|
|
||||||
merged := mergeDeviceLists(existing, scanned, configured)
|
|
||||||
require.Len(t, merged, 2)
|
|
||||||
byKey := make(map[deviceKey]*DeviceInfo, len(merged))
|
|
||||||
for _, device := range merged {
|
|
||||||
byKey[makeDeviceKey(device.Name, device.Type)] = device
|
|
||||||
assert.True(t, device.typeVerified, device.Type)
|
|
||||||
assert.Equal(t, "sat", device.parserType, device.Type)
|
|
||||||
assert.True(t, device.explicitType, device.Type)
|
|
||||||
}
|
|
||||||
assert.Contains(t, byKey, makeDeviceKey("/dev/sdb", "jms56x,0"))
|
|
||||||
assert.Contains(t, byKey, makeDeviceKey("/dev/sdb", "jms56x,1"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMergeDeviceListsDeduplicatesConfiguredIdentityAfterRekey(t *testing.T) {
|
|
||||||
scanned := []*DeviceInfo{{Name: "/dev/sdb", Type: "sat"}}
|
|
||||||
configured := []*DeviceInfo{
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,0", explicitType: true},
|
|
||||||
{Name: "/dev/sdb", Type: "jms56x,0", explicitType: true},
|
|
||||||
}
|
|
||||||
|
|
||||||
merged := mergeDeviceLists(nil, scanned, configured)
|
|
||||||
require.Len(t, merged, 1)
|
|
||||||
assert.Equal(t, "/dev/sdb", merged[0].Name)
|
|
||||||
assert.Equal(t, "jms56x,0", merged[0].Type)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMergeDeviceListsPreservesVerification(t *testing.T) {
|
func TestMergeDeviceListsPreservesVerification(t *testing.T) {
|
||||||
existing := []*DeviceInfo{
|
existing := []*DeviceInfo{
|
||||||
{Name: "/dev/sda", Type: "sat+megaraid", parserType: "sat", typeVerified: true},
|
{Name: "/dev/sda", Type: "sat+megaraid", parserType: "sat", typeVerified: true},
|
||||||
@@ -849,20 +753,6 @@ func TestParseSmartOutputKeepsCustomType(t *testing.T) {
|
|||||||
assert.Equal(t, "sat+megaraid", device.Type)
|
assert.Equal(t, "sat+megaraid", device.Type)
|
||||||
assert.Equal(t, "sat", device.parserType)
|
assert.Equal(t, "sat", device.parserType)
|
||||||
assert.True(t, device.typeVerified)
|
assert.True(t, device.typeVerified)
|
||||||
assert.Equal(t, "sat+megaraid", sm.SmartDataMap["9C40918040082"].DiskType)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseSmartOutputDoesNotNormalizeDeviceIdentity(t *testing.T) {
|
|
||||||
fixturePath := filepath.Join("test-data", "smart", "sda.json")
|
|
||||||
data, err := os.ReadFile(fixturePath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
|
||||||
device := &DeviceInfo{Name: "/dev/sda", Type: "ata", explicitType: true}
|
|
||||||
|
|
||||||
require.True(t, sm.parseSmartOutput(device, data))
|
|
||||||
assert.Equal(t, "sat", device.parserType)
|
|
||||||
assert.Equal(t, "ata", sm.SmartDataMap["9C40918040082"].DiskType)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseSmartOutputResetsVerificationOnFailure(t *testing.T) {
|
func TestParseSmartOutputResetsVerificationOnFailure(t *testing.T) {
|
||||||
@@ -1410,7 +1300,7 @@ func TestParseSmartForNvmeAppleSSD(t *testing.T) {
|
|||||||
darwinNvmeProvider: fakeProvider,
|
darwinNvmeProvider: fakeProvider,
|
||||||
}
|
}
|
||||||
|
|
||||||
hasData, _ := sm.parseSmartForNvme(data, "")
|
hasData, _ := sm.parseSmartForNvme(data)
|
||||||
require.True(t, hasData)
|
require.True(t, hasData)
|
||||||
|
|
||||||
deviceData, ok := sm.SmartDataMap["0ba0147940253c15"]
|
deviceData, ok := sm.SmartDataMap["0ba0147940253c15"]
|
||||||
@@ -1422,7 +1312,7 @@ func TestParseSmartForNvmeAppleSSD(t *testing.T) {
|
|||||||
assert.Equal(t, 1, providerCalls, "system_profiler should be called once")
|
assert.Equal(t, 1, providerCalls, "system_profiler should be called once")
|
||||||
|
|
||||||
// Second parse: provider should NOT be called again (cache hit)
|
// Second parse: provider should NOT be called again (cache hit)
|
||||||
_, _ = sm.parseSmartForNvme(data, "")
|
_, _ = sm.parseSmartForNvme(data)
|
||||||
assert.Equal(t, 1, providerCalls, "system_profiler should not be called again after caching")
|
assert.Equal(t, 1, providerCalls, "system_profiler should not be called again after caching")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,462 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"log/slog"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/btrfs"
|
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
zfsentity "github.com/henrygd/beszel/internal/entities/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
// zfsDatasetUsage holds usage values for a ZFS dataset mountpoint.
|
|
||||||
type zfsDatasetUsage struct {
|
|
||||||
used uint64
|
|
||||||
avail uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// datasetUsageRefreshInterval controls how often `zfs list` is re-run for the
|
|
||||||
// mountpoint usage map. Dataset inventory changes rarely.
|
|
||||||
const datasetUsageRefreshInterval = 5 * time.Minute
|
|
||||||
|
|
||||||
// poolStatsRefreshInterval controls how often `zpool list` is re-run for pool
|
|
||||||
// capacity. Health and I/O are read from procfs on Linux, so the utility only
|
|
||||||
// needs to refresh slow-moving space accounting.
|
|
||||||
const poolStatsRefreshInterval = time.Minute
|
|
||||||
|
|
||||||
// btrfsFilesystems is the btrfs source; overridable in tests.
|
|
||||||
var btrfsFilesystems = btrfs.Filesystems
|
|
||||||
|
|
||||||
type poolKernelSample struct {
|
|
||||||
nread uint64
|
|
||||||
nwrite uint64
|
|
||||||
at time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// StoragePoolManager combines independent backend inventories. Metrics and
|
|
||||||
// dataset usage require the agent lock; GetDetail is safe for concurrent calls.
|
|
||||||
type StoragePoolManager struct {
|
|
||||||
backends []*poolBackend
|
|
||||||
detailInterval time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// poolBackend owns one backend's collectors and caches. Collector functions
|
|
||||||
// are immutable after construction and may run concurrently for metrics/details.
|
|
||||||
type poolBackend struct {
|
|
||||||
name string
|
|
||||||
poolStatsFn func() ([]zfs.PoolStat, error) // capacity/health source
|
|
||||||
datasetsFn func() ([]zfs.Dataset, error) // dataset inventory source
|
|
||||||
kernelStatsFn func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source
|
|
||||||
poolStatusesFn func() ([]zfs.PoolStatus, error) // scrub/vdev detail source
|
|
||||||
|
|
||||||
poolData []zfs.PoolStat // cached pool inventory (TTL below)
|
|
||||||
lastPoolStats time.Time
|
|
||||||
kernelSamples map[string]poolKernelSample
|
|
||||||
|
|
||||||
datasetUsage map[string]zfsDatasetUsage // mountpoint -> usage
|
|
||||||
lastUsageRefresh time.Time
|
|
||||||
|
|
||||||
// Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on
|
|
||||||
// an interval. Accessed from handler goroutines, so it is mutex-protected.
|
|
||||||
detailMu sync.Mutex
|
|
||||||
detail *zfsentity.ZfsData
|
|
||||||
lastDetailRefresh time.Time
|
|
||||||
detailFailed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func newStoragePoolManager() *StoragePoolManager {
|
|
||||||
return &StoragePoolManager{
|
|
||||||
backends: []*poolBackend{newZfsBackend(), newBtrfsBackend()},
|
|
||||||
detailInterval: time.Hour,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newZfsBackend() *poolBackend {
|
|
||||||
return &poolBackend{
|
|
||||||
name: "zfs",
|
|
||||||
poolStatsFn: optionalPoolSource(zfs.PoolStats),
|
|
||||||
datasetsFn: optionalPoolSource(zfs.Datasets),
|
|
||||||
kernelStatsFn: optionalPoolSource(zfs.PoolKernelStats),
|
|
||||||
poolStatusesFn: optionalPoolSource(zfs.PoolStatuses),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newBtrfsBackend() *poolBackend {
|
|
||||||
return &poolBackend{
|
|
||||||
name: "btrfs",
|
|
||||||
poolStatsFn: btrfsSource(btrfsPoolStats),
|
|
||||||
kernelStatsFn: btrfsSource(btrfsKernelStats),
|
|
||||||
poolStatusesFn: btrfsSource(btrfsPoolStatuses),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// datasets is optional: only backends that expose datasets provide a collector.
|
|
||||||
func (b *poolBackend) datasets() ([]zfs.Dataset, error) {
|
|
||||||
if b.datasetsFn == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return b.datasetsFn()
|
|
||||||
}
|
|
||||||
|
|
||||||
// A missing utility/interface is a successfully observed absent backend.
|
|
||||||
func optionalPoolSource[T any](source func() ([]T, error)) func() ([]T, error) {
|
|
||||||
return func() ([]T, error) {
|
|
||||||
items, err := source()
|
|
||||||
if errors.Is(err, zfs.ErrNoZfs) || errors.Is(err, exec.ErrNotFound) || errors.Is(err, errors.ErrUnsupported) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return items, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsSource[T any](convert func(btrfs.Filesystem) T) func() ([]T, error) {
|
|
||||||
return func() ([]T, error) {
|
|
||||||
filesystems, err := optionalPoolSource(btrfsFilesystems)()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
items := make([]T, 0, len(filesystems))
|
|
||||||
for _, fs := range filesystems {
|
|
||||||
items = append(items, convert(fs))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update refreshes systemStats.ZfsPools with the latest pool data. I/O
|
|
||||||
// throughput and health come from inexpensive kernel kstats on Linux. Pool
|
|
||||||
// capacity and dataset usage come from separately cached utility calls. The
|
|
||||||
// pool map is empty when both backends are absent.
|
|
||||||
func (m *StoragePoolManager) Update(systemStats *system.Stats) {
|
|
||||||
// Rebuild the combined map so successful pool removals clear old samples.
|
|
||||||
systemStats.ZfsPools = nil
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
backend.updateBackendStats(systemStats)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *poolBackend) updateBackendStats(systemStats *system.Stats) {
|
|
||||||
pools := b.poolStats()
|
|
||||||
if len(pools) == 0 {
|
|
||||||
b.kernelSamples = nil
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
kernelStats, ioRates := b.kernelStats()
|
|
||||||
|
|
||||||
if systemStats.ZfsPools == nil {
|
|
||||||
systemStats.ZfsPools = make(map[string]*system.ZfsPool, len(pools))
|
|
||||||
}
|
|
||||||
for i := range pools {
|
|
||||||
pool := &pools[i]
|
|
||||||
// Full precision, matching the dataset values below; the frontend
|
|
||||||
// formats any magnitude.
|
|
||||||
stats := &system.ZfsPool{
|
|
||||||
DisplayName: pool.DisplayName,
|
|
||||||
Raw: pool.Raw,
|
|
||||||
Total: float64(pool.Size) / (1024 * 1024 * 1024),
|
|
||||||
Used: float64(pool.Alloc) / (1024 * 1024 * 1024),
|
|
||||||
Health: pool.Health,
|
|
||||||
}
|
|
||||||
if kernel, exists := kernelStats[pool.Name]; exists && kernel.Health != "" {
|
|
||||||
stats.Health = kernel.Health
|
|
||||||
}
|
|
||||||
if io, exists := ioRates[pool.Name]; exists {
|
|
||||||
stats.ReadBytes = io.NRead
|
|
||||||
stats.WriteBytes = io.NWrite
|
|
||||||
}
|
|
||||||
slog.Debug("Storage pool sample", "backend", b.name, "pool", pool.Name, "health", stats.Health, "used_gb", stats.Used, "read_bps", stats.ReadBytes, "write_bps", stats.WriteBytes)
|
|
||||||
systemStats.ZfsPools[pool.Name] = stats
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// poolStats returns the cached pool inventory, calling its collector at most
|
|
||||||
// every poolStatsRefreshInterval. On failure the previous inventory is
|
|
||||||
// retained and the refresh is retried on the next cadence.
|
|
||||||
func (b *poolBackend) poolStats() []zfs.PoolStat {
|
|
||||||
if b.lastPoolStats.IsZero() || time.Since(b.lastPoolStats) >= poolStatsRefreshInterval {
|
|
||||||
pools, err := b.poolStatsFn()
|
|
||||||
if err != nil {
|
|
||||||
slog.Debug("Storage pool stats unavailable", "backend", b.name, "err", err)
|
|
||||||
} else {
|
|
||||||
b.poolData = pools
|
|
||||||
}
|
|
||||||
b.lastPoolStats = time.Now()
|
|
||||||
}
|
|
||||||
return b.poolData
|
|
||||||
}
|
|
||||||
|
|
||||||
// kernelStats reads cumulative pool counters and converts them to per-second
|
|
||||||
// rates. Counter decreases indicate a pool export/import and reset the
|
|
||||||
// baseline instead of producing an underflow spike.
|
|
||||||
func (b *poolBackend) kernelStats() (map[string]zfs.PoolKernelStat, map[string]zfs.PoolIoStats) {
|
|
||||||
if b.kernelStatsFn == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
stats, err := b.kernelStatsFn()
|
|
||||||
if err != nil {
|
|
||||||
slog.Debug("Storage pool kernel stats unavailable", "backend", b.name, "err", err)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
byName := make(map[string]zfs.PoolKernelStat, len(stats))
|
|
||||||
rates := make(map[string]zfs.PoolIoStats, len(stats))
|
|
||||||
nextSamples := make(map[string]poolKernelSample, len(stats))
|
|
||||||
for _, stat := range stats {
|
|
||||||
byName[stat.Name] = stat
|
|
||||||
if previous, ok := b.kernelSamples[stat.Name]; ok && now.After(previous.at) &&
|
|
||||||
stat.NRead >= previous.nread && stat.NWrite >= previous.nwrite {
|
|
||||||
seconds := now.Sub(previous.at).Seconds()
|
|
||||||
rates[stat.Name] = zfs.PoolIoStats{
|
|
||||||
NRead: uint64(float64(stat.NRead-previous.nread) / seconds),
|
|
||||||
NWrite: uint64(float64(stat.NWrite-previous.nwrite) / seconds),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
nextSamples[stat.Name] = poolKernelSample{nread: stat.NRead, nwrite: stat.NWrite, at: now}
|
|
||||||
}
|
|
||||||
b.kernelSamples = nextSamples
|
|
||||||
return byName, rates
|
|
||||||
}
|
|
||||||
|
|
||||||
// refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed
|
|
||||||
// and rebuilds the mountpoint-keyed usage map.
|
|
||||||
func (b *poolBackend) refreshDatasetUsage() {
|
|
||||||
if !b.lastUsageRefresh.IsZero() && time.Since(b.lastUsageRefresh) < datasetUsageRefreshInterval {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
datasets, err := b.datasets()
|
|
||||||
if err != nil {
|
|
||||||
slog.Debug("Storage pool dataset usage unavailable", "backend", b.name, "err", err)
|
|
||||||
} else {
|
|
||||||
usage := make(map[string]zfsDatasetUsage, len(datasets))
|
|
||||||
for _, ds := range datasets {
|
|
||||||
if ds.Mountpoint != "" && ds.Mountpoint != "-" {
|
|
||||||
usage[ds.Mountpoint] = zfsDatasetUsage{used: ds.Used, avail: ds.Avail}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.datasetUsage = usage
|
|
||||||
}
|
|
||||||
b.lastUsageRefresh = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
// DatasetUsage returns ZFS dataset usage keyed by mountpoint, refreshed at
|
|
||||||
// most every datasetUsageRefreshInterval. On failure the previous map is
|
|
||||||
// retained and a debug log is emitted.
|
|
||||||
func (m *StoragePoolManager) DatasetUsage() map[string]zfsDatasetUsage {
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
if backend.name == "zfs" {
|
|
||||||
backend.refreshDatasetUsage()
|
|
||||||
return backend.datasetUsage
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetDetail combines backend snapshots, identifying successful inventories so
|
|
||||||
// the hub can accept partial updates without deleting failed backend records.
|
|
||||||
func (m *StoragePoolManager) GetDetail(force bool) *zfsentity.ZfsData {
|
|
||||||
data := &zfsentity.ZfsData{Complete: true}
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
snapshot := backend.getBackendDetail(force, m.detailInterval)
|
|
||||||
data.Pools = append(data.Pools, snapshot.Pools...)
|
|
||||||
if snapshot.Complete {
|
|
||||||
data.CompleteBackends = append(data.CompleteBackends, backend.name)
|
|
||||||
} else {
|
|
||||||
data.Complete = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *poolBackend) getBackendDetail(force bool, interval time.Duration) *zfsentity.ZfsData {
|
|
||||||
b.detailMu.Lock()
|
|
||||||
defer b.detailMu.Unlock()
|
|
||||||
|
|
||||||
if force || b.detailFailed || b.detail == nil || time.Since(b.lastDetailRefresh) >= interval {
|
|
||||||
if data, err := b.collectDetail(b.detail); err != nil {
|
|
||||||
b.detailFailed = true
|
|
||||||
slog.Debug("Storage pool detail collection failed", "backend", b.name, "err", err)
|
|
||||||
if b.detail == nil {
|
|
||||||
return &zfsentity.ZfsData{}
|
|
||||||
}
|
|
||||||
return &zfsentity.ZfsData{Pools: b.detail.Pools}
|
|
||||||
} else {
|
|
||||||
b.detailFailed = false
|
|
||||||
b.detail = data
|
|
||||||
b.lastDetailRefresh = time.Now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if b.detail == nil {
|
|
||||||
return &zfsentity.ZfsData{}
|
|
||||||
}
|
|
||||||
return b.detail
|
|
||||||
}
|
|
||||||
|
|
||||||
// collectDetail builds a ZfsData payload from the current system state.
|
|
||||||
func (b *poolBackend) collectDetail(previous *zfsentity.ZfsData) (*zfsentity.ZfsData, error) {
|
|
||||||
pools, err := b.poolStatsFn()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(pools) == 0 {
|
|
||||||
return &zfsentity.ZfsData{Pools: []*zfsentity.PoolDetail{}, Complete: true}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
statuses, statusErr := b.poolStatusesFn()
|
|
||||||
if statusErr != nil {
|
|
||||||
slog.Debug("Storage pool status unavailable", "backend", b.name, "err", statusErr)
|
|
||||||
}
|
|
||||||
datasets, datasetsErr := b.datasets()
|
|
||||||
if datasetsErr != nil {
|
|
||||||
slog.Debug("Storage pool datasets unavailable", "backend", b.name, "err", datasetsErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
statusByPool := make(map[string]zfs.PoolStatus, len(statuses))
|
|
||||||
for _, st := range statuses {
|
|
||||||
statusByPool[st.Name] = st
|
|
||||||
}
|
|
||||||
|
|
||||||
previousByPool := make(map[string]*zfsentity.PoolDetail)
|
|
||||||
if previous != nil {
|
|
||||||
for _, pool := range previous.Pools {
|
|
||||||
if pool != nil {
|
|
||||||
previousByPool[pool.Name] = pool
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
data := &zfsentity.ZfsData{Pools: make([]*zfsentity.PoolDetail, 0, len(pools)), Complete: true}
|
|
||||||
for i := range pools {
|
|
||||||
p := &pools[i]
|
|
||||||
detail := &zfsentity.PoolDetail{
|
|
||||||
DisplayName: p.DisplayName,
|
|
||||||
Raw: p.Raw,
|
|
||||||
Name: p.Name,
|
|
||||||
Health: p.Health,
|
|
||||||
Size: p.Size,
|
|
||||||
Alloc: p.Alloc,
|
|
||||||
Free: p.Free,
|
|
||||||
}
|
|
||||||
if st, ok := statusByPool[p.Name]; statusErr == nil && ok {
|
|
||||||
if st.Scrub.State != "" && st.Scrub.State != "NONE" {
|
|
||||||
detail.Scrub = &zfsentity.Scrub{
|
|
||||||
State: st.Scrub.State,
|
|
||||||
Progress: st.Scrub.Progress,
|
|
||||||
Errors: st.Scrub.Errors,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, v := range st.Vdevs {
|
|
||||||
detail.Vdevs = append(detail.Vdevs, &zfsentity.Vdev{
|
|
||||||
Name: v.Name,
|
|
||||||
State: v.State,
|
|
||||||
ReadErrs: v.ReadErrs,
|
|
||||||
WriteErrs: v.WriteErrs,
|
|
||||||
ChecksumErrs: v.ChecksumErrs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if cached := previousByPool[p.Name]; cached != nil {
|
|
||||||
detail.Scrub = cached.Scrub
|
|
||||||
detail.Vdevs = cached.Vdevs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if datasetsErr == nil {
|
|
||||||
foundDataset := false
|
|
||||||
for _, ds := range datasets {
|
|
||||||
if poolOfDataset(ds.Name) == p.Name {
|
|
||||||
foundDataset = true
|
|
||||||
detail.Datasets = append(detail.Datasets, &zfsentity.Dataset{
|
|
||||||
Name: ds.Name,
|
|
||||||
Used: ds.Used,
|
|
||||||
Avail: ds.Avail,
|
|
||||||
Mountpoint: ds.Mountpoint,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !foundDataset {
|
|
||||||
if cached := previousByPool[p.Name]; cached != nil {
|
|
||||||
detail.Datasets = cached.Datasets
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if cached := previousByPool[p.Name]; cached != nil {
|
|
||||||
detail.Datasets = cached.Datasets
|
|
||||||
}
|
|
||||||
data.Pools = append(data.Pools, detail)
|
|
||||||
}
|
|
||||||
return data, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// poolOfDataset returns the pool name for a dataset name (everything before
|
|
||||||
// the first '/'). Datasets without a separator belong to a pool of the same
|
|
||||||
// name.
|
|
||||||
func poolOfDataset(name string) string {
|
|
||||||
if idx := strings.IndexByte(name, '/'); idx >= 0 {
|
|
||||||
return name[:idx]
|
|
||||||
}
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
// ZfsMountpoints returns the set of mountpoints backed by ZFS datasets.
|
|
||||||
func (m *StoragePoolManager) ZfsMountpoints() map[string]bool {
|
|
||||||
usage := m.DatasetUsage()
|
|
||||||
mountpoints := make(map[string]bool, len(usage))
|
|
||||||
for mountpoint := range usage {
|
|
||||||
mountpoints[mountpoint] = true
|
|
||||||
}
|
|
||||||
return mountpoints
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsPoolStats(fs btrfs.Filesystem) zfs.PoolStat {
|
|
||||||
return zfs.PoolStat{MountID: fs.MountID, IODevice: fs.IODevice, Raw: fs.Raw, DisplayName: fs.Name, Name: "b:" + fs.UUID, Size: fs.Size, Alloc: fs.Alloc, Free: fs.Size - min(fs.Alloc, fs.Size), Health: fs.Health}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsKernelStats(fs btrfs.Filesystem) zfs.PoolKernelStat {
|
|
||||||
return zfs.PoolKernelStat{Name: "b:" + fs.UUID, Health: fs.Health, NRead: fs.NRead, NWrite: fs.NWrite}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsPoolStatuses(fs btrfs.Filesystem) zfs.PoolStatus {
|
|
||||||
status := zfs.PoolStatus{Name: "b:" + fs.UUID, State: fs.Health, Scrub: zfs.ScrubStatus{State: "NONE"}}
|
|
||||||
for _, dev := range fs.Devices {
|
|
||||||
status.Vdevs = append(status.Vdevs, zfs.VdevStatus{
|
|
||||||
Name: dev.Name, State: dev.State,
|
|
||||||
ReadErrs: dev.ReadErrs, WriteErrs: dev.WriteErrs, ChecksumErrs: dev.CorruptionErrs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return status
|
|
||||||
}
|
|
||||||
|
|
||||||
// markDuplicateCharts leaves pool telemetry and detail intact, but tells the
|
|
||||||
// hub which charts already have a filesystem equivalent. Only exact kernel
|
|
||||||
// filesystem and I/O-device matches qualify; labels are never used.
|
|
||||||
func (m *StoragePoolManager) markDuplicateCharts(stats *system.Stats, filesystems map[string]*system.FsStats, mountID func(string) string) {
|
|
||||||
identities := make(map[string]string, len(filesystems))
|
|
||||||
for device, fs := range filesystems {
|
|
||||||
if fs.DiskTotal > 0 {
|
|
||||||
identities[device] = mountID(fs.Mountpoint)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
for _, pool := range backend.poolData {
|
|
||||||
sample := stats.ZfsPools[pool.Name]
|
|
||||||
if sample == nil || pool.MountID == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for device, identity := range identities {
|
|
||||||
if identity != pool.MountID {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Raw physical usage is not equivalent to a filesystem usage chart.
|
|
||||||
sample.HideUsage = !pool.Raw
|
|
||||||
if pool.IODevice != "" && pool.IODevice == device {
|
|
||||||
sample.HideIO = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,520 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/btrfs"
|
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestOptionalPoolSource(t *testing.T) {
|
|
||||||
failure := errors.New("timeout")
|
|
||||||
for _, err := range []error{nil, zfs.ErrNoZfs, fmt.Errorf("zpool: %w", exec.ErrNotFound), errors.ErrUnsupported, failure} {
|
|
||||||
_, got := optionalPoolSource(func() ([]zfs.PoolStat, error) { return nil, err })()
|
|
||||||
if err == failure {
|
|
||||||
assert.ErrorIs(t, got, failure)
|
|
||||||
} else {
|
|
||||||
assert.NoError(t, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type poolTestBackend struct {
|
|
||||||
name string
|
|
||||||
err error
|
|
||||||
alloc uint64
|
|
||||||
read uint64
|
|
||||||
empty bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (state *poolTestBackend) backend() *poolBackend {
|
|
||||||
name := "zfs"
|
|
||||||
if strings.HasPrefix(state.name, "b:") {
|
|
||||||
name = "btrfs"
|
|
||||||
}
|
|
||||||
return &poolBackend{
|
|
||||||
name: name,
|
|
||||||
poolStatsFn: func() ([]zfs.PoolStat, error) {
|
|
||||||
if state.empty {
|
|
||||||
return nil, state.err
|
|
||||||
}
|
|
||||||
return []zfs.PoolStat{{Name: state.name, Size: 100, Alloc: state.alloc}}, state.err
|
|
||||||
},
|
|
||||||
kernelStatsFn: func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
return []zfs.PoolKernelStat{{Name: state.name, NRead: state.read}}, state.err
|
|
||||||
},
|
|
||||||
poolStatusesFn: func() ([]zfs.PoolStatus, error) { return nil, nil },
|
|
||||||
datasetsFn: func() ([]zfs.Dataset, error) { return nil, nil },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIndependentPoolBackendCaches(t *testing.T) {
|
|
||||||
for _, failed := range []int{0, 1} {
|
|
||||||
t.Run([]string{"zfs", "btrfs"}[failed], func(t *testing.T) {
|
|
||||||
states := []*poolTestBackend{{name: "tank", alloc: 10}, {name: "b:uuid", alloc: 10}}
|
|
||||||
managers := []*poolBackend{states[0].backend(), states[1].backend()}
|
|
||||||
zm := &StoragePoolManager{backends: managers, detailInterval: time.Hour}
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 2)
|
|
||||||
require.True(t, zm.GetDetail(true).Complete)
|
|
||||||
baseline := poolKernelSample{at: time.Now().Add(-time.Second)}
|
|
||||||
for i, m := range managers {
|
|
||||||
m.lastPoolStats = time.Time{}
|
|
||||||
m.kernelSamples[states[i].name] = baseline
|
|
||||||
states[i].alloc = 20
|
|
||||||
states[i].read = 100
|
|
||||||
}
|
|
||||||
states[failed].err = errors.New("collection failed")
|
|
||||||
zm.Update(&stats)
|
|
||||||
healthy := 1 - failed
|
|
||||||
assert.Equal(t, uint64(10), managers[failed].poolData[0].Alloc)
|
|
||||||
assert.Equal(t, uint64(20), managers[healthy].poolData[0].Alloc)
|
|
||||||
assert.Equal(t, baseline, managers[failed].kernelSamples[states[failed].name])
|
|
||||||
assert.Zero(t, stats.ZfsPools[states[failed].name].ReadBytes)
|
|
||||||
assert.Positive(t, stats.ZfsPools[states[healthy].name].ReadBytes)
|
|
||||||
partial := zm.GetDetail(true)
|
|
||||||
assert.False(t, partial.Complete)
|
|
||||||
assert.False(t, partial.CanRefreshPool(states[failed].name))
|
|
||||||
assert.True(t, partial.CanRefreshPool(states[healthy].name))
|
|
||||||
assert.Equal(t, uint64(10), partial.Pools[failed].Alloc)
|
|
||||||
assert.Equal(t, uint64(20), partial.Pools[healthy].Alloc)
|
|
||||||
assert.False(t, zm.GetDetail(false).Complete, "a failed forced refresh must not become complete from cache")
|
|
||||||
|
|
||||||
// Successful empty inventory removes only the healthy backend's pool.
|
|
||||||
states[healthy].empty = true
|
|
||||||
managers[healthy].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 1)
|
|
||||||
assert.Contains(t, stats.ZfsPools, states[failed].name)
|
|
||||||
partial = zm.GetDetail(true)
|
|
||||||
require.Len(t, partial.Pools, 1)
|
|
||||||
assert.True(t, partial.CanRefreshPool(states[healthy].name))
|
|
||||||
|
|
||||||
// Recovery uses the retained I/O baseline, then normal removal works.
|
|
||||||
states[failed].err = nil
|
|
||||||
managers[failed].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Positive(t, stats.ZfsPools[states[failed].name].ReadBytes)
|
|
||||||
assert.True(t, zm.GetDetail(true).Complete)
|
|
||||||
states[failed].empty = true
|
|
||||||
managers[failed].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Empty(t, stats.ZfsPools)
|
|
||||||
assert.Empty(t, zm.GetDetail(true).Pools)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIndependentBackendsWithoutCache(t *testing.T) {
|
|
||||||
z := &poolTestBackend{name: "tank", err: errors.New("ZFS failure")}
|
|
||||||
b := &poolTestBackend{name: "b:uuid", alloc: 20}
|
|
||||||
zm := &StoragePoolManager{backends: []*poolBackend{z.backend(), b.backend()}, detailInterval: time.Hour}
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 1)
|
|
||||||
assert.Contains(t, stats.ZfsPools, "b:uuid")
|
|
||||||
detail := zm.GetDetail(true)
|
|
||||||
require.Len(t, detail.Pools, 1)
|
|
||||||
assert.False(t, detail.Complete)
|
|
||||||
assert.Equal(t, []string{"btrfs"}, detail.CompleteBackends)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConcurrentBackendDetailsAndMetrics(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{backends: []*poolBackend{(&poolTestBackend{name: "tank"}).backend(), (&poolTestBackend{name: "b:uuid"}).backend()}, detailInterval: time.Hour}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for i := 0; i < 3; i++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func(metrics bool) {
|
|
||||||
defer wg.Done()
|
|
||||||
for j := 0; j < 10; j++ {
|
|
||||||
if metrics {
|
|
||||||
zm.Update(&system.Stats{})
|
|
||||||
} else {
|
|
||||||
zm.GetDetail(true)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}(i == 0)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStoragePoolBackendOrder(t *testing.T) {
|
|
||||||
z := (&poolTestBackend{name: "tank"}).backend()
|
|
||||||
b := (&poolTestBackend{name: "b:uuid"}).backend()
|
|
||||||
b.datasetsFn = nil // Btrfs does not expose datasets.
|
|
||||||
z.datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{{Name: "tank/data", Mountpoint: "/tank", Used: 10}}, nil
|
|
||||||
}
|
|
||||||
m := &StoragePoolManager{backends: []*poolBackend{b, z}, detailInterval: time.Hour}
|
|
||||||
var stats system.Stats
|
|
||||||
m.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 2)
|
|
||||||
detail := m.GetDetail(true)
|
|
||||||
require.True(t, detail.Complete)
|
|
||||||
assert.Equal(t, []string{"btrfs", "zfs"}, detail.CompleteBackends)
|
|
||||||
assert.Empty(t, detail.Pools[0].Datasets)
|
|
||||||
assert.Len(t, detail.Pools[1].Datasets, 1)
|
|
||||||
assert.Equal(t, uint64(10), m.DatasetUsage()["/tank"].used)
|
|
||||||
|
|
||||||
b.poolData[0].MountID = "uuid"
|
|
||||||
b.poolData[0].IODevice = "sda"
|
|
||||||
calls := 0
|
|
||||||
m.markDuplicateCharts(&stats, map[string]*system.FsStats{
|
|
||||||
"sda": {Mountpoint: "/", DiskTotal: 100},
|
|
||||||
}, func(string) string { calls++; return "uuid" })
|
|
||||||
assert.Equal(t, 1, calls, "resolve each filesystem once across all backends")
|
|
||||||
assert.True(t, stats.ZfsPools["b:uuid"].HideUsage)
|
|
||||||
assert.True(t, stats.ZfsPools["b:uuid"].HideIO)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdatePopulatesZfsPools(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "DEGRADED"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "tank/apps", Used: 5000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
|
||||||
{Name: "tank/backup", Used: 6000000000000, Avail: 11999000000000, Mountpoint: "/tank/backup"},
|
|
||||||
// Small zvol (Proxmox VM EFI disk): must not round to zero.
|
|
||||||
{Name: "rpool/vm-100-disk-2", Used: 4194304, Avail: 0, Mountpoint: "-"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
var kernelCalls int
|
|
||||||
zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
kernelCalls++
|
|
||||||
return []zfs.PoolKernelStat{{
|
|
||||||
Name: "tank", Health: "ONLINE",
|
|
||||||
NRead: uint64(kernelCalls-1) * 1250, NWrite: uint64(kernelCalls-1) * 5120,
|
|
||||||
}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
// The first kernel sample establishes the cumulative-counter baseline.
|
|
||||||
zm.Update(&stats)
|
|
||||||
zm.backends[0].kernelSamples["tank"] = poolKernelSample{at: time.Now().Add(-time.Second)}
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.NotNil(t, stats.ZfsPools)
|
|
||||||
require.Contains(t, stats.ZfsPools, "tank")
|
|
||||||
assert.InDelta(t, 22350.8105, stats.ZfsPools["tank"].Total, 0.0001) // Size in GiB
|
|
||||||
assert.InDelta(t, 11175.8709, stats.ZfsPools["tank"].Used, 0.0001) // Alloc in GiB
|
|
||||||
assert.Equal(t, "ONLINE", stats.ZfsPools["tank"].Health)
|
|
||||||
assert.InDelta(t, 1250, stats.ZfsPools["tank"].ReadBytes, 5)
|
|
||||||
assert.InDelta(t, 5120, stats.ZfsPools["tank"].WriteBytes, 5)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdateKernelStatsMissing verifies pools without a kernel sample report zero
|
|
||||||
// I/O instead of erroring.
|
|
||||||
func TestUpdateKernelStatsMissing(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Size: 1, Alloc: 1, Health: "ONLINE"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.NotNil(t, stats.ZfsPools)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateKernelCounterReset(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Health: "ONLINE"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
zm.backends[0].kernelSamples = map[string]poolKernelSample{
|
|
||||||
"tank": {nread: 100, nwrite: 200, at: time.Now().Add(-time.Second)},
|
|
||||||
}
|
|
||||||
zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
return []zfs.PoolKernelStat{{Name: "tank", Health: "ONLINE", NRead: 10, NWrite: 20}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateNoZfs(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
calls := 0
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
calls++
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Nil(t, stats.ZfsPools)
|
|
||||||
assert.Equal(t, 1, calls, "failed pool discovery should be cached until the next refresh interval")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateEmptyPools(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
calls := 0
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
calls++
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Nil(t, stats.ZfsPools)
|
|
||||||
assert.Equal(t, 1, calls, "an empty pool inventory should be cached until the next refresh interval")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetUsage(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
calls := 0
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
calls++
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
|
||||||
{Name: "tank/apps", Used: 1000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
|
||||||
{Name: "rpool", Used: 900000000000, Avail: 300000000000, Mountpoint: "-"}, // zvol/unmounted: excluded
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
usage := zm.DatasetUsage()
|
|
||||||
require.Len(t, usage, 2)
|
|
||||||
assert.Equal(t, zfsDatasetUsage{used: 12000000000000, avail: 11999000000000}, usage["/tank"])
|
|
||||||
assert.Equal(t, zfsDatasetUsage{used: 1000000000000, avail: 11999000000000}, usage["/tank/apps"])
|
|
||||||
assert.Equal(t, 1, calls)
|
|
||||||
|
|
||||||
// Second call within the refresh window must not re-run the collector.
|
|
||||||
zm.DatasetUsage()
|
|
||||||
assert.Equal(t, 1, calls)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetUsageRefreshOnErrorKeepsPrevious(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{{Name: "tank", Used: 1, Avail: 1, Mountpoint: "/tank"}}, nil
|
|
||||||
}
|
|
||||||
assert.Len(t, zm.DatasetUsage(), 1)
|
|
||||||
|
|
||||||
// Force refresh window expiry, then a failing collector.
|
|
||||||
zm.backends[0].lastUsageRefresh = time.Now().Add(-10 * time.Minute)
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
}
|
|
||||||
usage := zm.DatasetUsage()
|
|
||||||
assert.Len(t, usage, 1, "previous usage should be retained on error")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetUsageClearsAbsentBackend(t *testing.T) {
|
|
||||||
b := newZfsBackend()
|
|
||||||
b.datasetUsage = map[string]zfsDatasetUsage{"/tank": {used: 1, avail: 1}}
|
|
||||||
b.datasetsFn = optionalPoolSource(func() ([]zfs.Dataset, error) {
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
})
|
|
||||||
|
|
||||||
datasets, err := b.datasets()
|
|
||||||
require.NoError(t, err, "an absent backend must not produce an error to log")
|
|
||||||
assert.Empty(t, datasets)
|
|
||||||
b.refreshDatasetUsage()
|
|
||||||
assert.Empty(t, b.datasetUsage)
|
|
||||||
assert.False(t, b.lastUsageRefresh.IsZero())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDetailForceRefresh(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
poolCalls := 0
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
poolCalls++
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
|
|
||||||
first := zm.GetDetail(false)
|
|
||||||
assert.True(t, first.Complete)
|
|
||||||
require.Len(t, first.Pools, 1)
|
|
||||||
assert.Equal(t, uint64(1), first.Pools[0].Alloc)
|
|
||||||
|
|
||||||
cached := zm.GetDetail(false)
|
|
||||||
require.Len(t, cached.Pools, 1)
|
|
||||||
assert.Equal(t, uint64(1), cached.Pools[0].Alloc)
|
|
||||||
assert.Equal(t, 1, poolCalls)
|
|
||||||
|
|
||||||
refreshed := zm.GetDetail(true)
|
|
||||||
assert.True(t, refreshed.Complete)
|
|
||||||
require.Len(t, refreshed.Pools, 1)
|
|
||||||
assert.Equal(t, uint64(2), refreshed.Pools[0].Alloc)
|
|
||||||
assert.Equal(t, 2, poolCalls)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDetailSuccessfulEmptyInventoryClearsCache(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
|
|
||||||
require.Len(t, zm.GetDetail(false).Pools, 1)
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, nil }
|
|
||||||
empty := zm.GetDetail(true)
|
|
||||||
assert.True(t, empty.Complete)
|
|
||||||
assert.Empty(t, empty.Pools)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDetailFailureReturnsIncompleteCachedInventory(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) {
|
|
||||||
return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "mirror-0"}}}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{{Name: "tank/data"}}, nil
|
|
||||||
}
|
|
||||||
first := zm.GetDetail(false)
|
|
||||||
require.True(t, first.Complete)
|
|
||||||
require.Len(t, first.Pools[0].Vdevs, 1)
|
|
||||||
require.Len(t, first.Pools[0].Datasets, 1)
|
|
||||||
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, zfs.ErrNoZfs }
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, zfs.ErrNoZfs }
|
|
||||||
partial := zm.GetDetail(true)
|
|
||||||
require.True(t, partial.Complete)
|
|
||||||
require.Len(t, partial.Pools[0].Vdevs, 1)
|
|
||||||
require.Len(t, partial.Pools[0].Datasets, 1)
|
|
||||||
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, zfs.ErrNoZfs }
|
|
||||||
lastSuccessfulRefresh := zm.backends[0].lastDetailRefresh
|
|
||||||
failed := zm.GetDetail(true)
|
|
||||||
assert.False(t, failed.Complete)
|
|
||||||
require.Len(t, failed.Pools, 1)
|
|
||||||
assert.Equal(t, "tank", failed.Pools[0].Name)
|
|
||||||
assert.Equal(t, lastSuccessfulRefresh, zm.backends[0].lastDetailRefresh)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestZfsMountpoints(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "tank", Mountpoint: "/tank"},
|
|
||||||
{Name: "rpool/ROOT/pve-1", Mountpoint: "/"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
mountpoints := zm.ZfsMountpoints()
|
|
||||||
assert.Len(t, mountpoints, 2)
|
|
||||||
assert.True(t, mountpoints["/tank"])
|
|
||||||
assert.True(t, mountpoints["/"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBtrfsRawCapacityPropagates(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolStatsFn: func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{btrfsPoolStats(btrfs.Filesystem{UUID: "raw", Name: "raw", Size: 200, Alloc: 100, Raw: true})}, nil
|
|
||||||
},
|
|
||||||
poolStatusesFn: func() ([]zfs.PoolStatus, error) { return nil, nil },
|
|
||||||
datasetsFn: func() ([]zfs.Dataset, error) { return nil, nil }}}}
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.True(t, stats.ZfsPools["b:raw"].Raw)
|
|
||||||
detail := zm.GetDetail(true)
|
|
||||||
require.True(t, detail.Complete)
|
|
||||||
require.Len(t, detail.Pools, 1)
|
|
||||||
assert.True(t, detail.Pools[0].Raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMarkDuplicatePoolCharts(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, poolID, device string
|
|
||||||
raw bool
|
|
||||||
diskTotal float64
|
|
||||||
wantUsage, wantIO bool
|
|
||||||
}{
|
|
||||||
{"single device root", "fs1", "dm-0", false, 100, true, true},
|
|
||||||
{"multi device", "fs1", "", false, 100, true, false},
|
|
||||||
{"different IO device", "fs1", "nvme0n1", false, 100, true, false},
|
|
||||||
{"different filesystem", "fs2", "dm-0", false, 100, false, false},
|
|
||||||
{"unknown identity", "", "dm-0", false, 100, false, false},
|
|
||||||
{"raw usage", "fs1", "dm-0", true, 100, false, true},
|
|
||||||
{"failed disk collection", "fs1", "dm-0", false, 0, false, false},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolData: []zfs.PoolStat{{Name: "arbitrary label", MountID: tc.poolID, IODevice: tc.device, Raw: tc.raw}}}}}
|
|
||||||
stats := &system.Stats{ZfsPools: map[string]*system.ZfsPool{"arbitrary label": {}}}
|
|
||||||
fs := map[string]*system.FsStats{"dm-0": {Root: true, Mountpoint: "/", DiskTotal: tc.diskTotal}}
|
|
||||||
zm.markDuplicateCharts(stats, fs, func(string) string { return "fs1" })
|
|
||||||
assert.Equal(t, tc.wantUsage, stats.ZfsPools["arbitrary label"].HideUsage)
|
|
||||||
assert.Equal(t, tc.wantIO, stats.ZfsPools["arbitrary label"].HideIO)
|
|
||||||
// Bind mounts and custom extra-filesystem names have the same identity.
|
|
||||||
fs["dm-0"].Root = false
|
|
||||||
fs["dm-0"].Mountpoint = "/extra-filesystems/storage"
|
|
||||||
fs["dm-0"].Name = "custom name"
|
|
||||||
stats.ZfsPools["arbitrary label"] = &system.ZfsPool{}
|
|
||||||
zm.markDuplicateCharts(stats, fs, func(string) string { return "fs1" })
|
|
||||||
assert.Equal(t, tc.wantUsage, stats.ZfsPools["arbitrary label"].HideUsage)
|
|
||||||
assert.Equal(t, tc.wantIO, stats.ZfsPools["arbitrary label"].HideIO)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBtrfsPoolIdentities(t *testing.T) {
|
|
||||||
old := btrfsFilesystems
|
|
||||||
t.Cleanup(func() { btrfsFilesystems = old })
|
|
||||||
label := "tank"
|
|
||||||
btrfsFilesystems = func() ([]btrfs.Filesystem, error) {
|
|
||||||
return []btrfs.Filesystem{
|
|
||||||
{UUID: "11111111-1111-4111-8111-111111111111", Name: label, Size: 100, Health: "ONLINE", NRead: 100, Devices: []btrfs.Device{{Name: "first"}}},
|
|
||||||
{UUID: "22222222-2222-4222-8222-222222222222", Name: "tank", Size: 200, Health: "DEGRADED", NRead: 200, Devices: []btrfs.Device{{Name: "second"}}},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolStatsFn: func() ([]zfs.PoolStat, error) { return []zfs.PoolStat{{Name: "tank", Size: 300}}, nil },
|
|
||||||
kernelStatsFn: func() ([]zfs.PoolKernelStat, error) { return []zfs.PoolKernelStat{{Name: "tank", NRead: 300}}, nil },
|
|
||||||
poolStatusesFn: func() ([]zfs.PoolStatus, error) {
|
|
||||||
return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "zfs-device"}}}}, nil
|
|
||||||
},
|
|
||||||
|
|
||||||
datasetsFn: func() ([]zfs.Dataset, error) { return []zfs.Dataset{{Name: "tank/data"}}, nil }}, newBtrfsBackend()}}
|
|
||||||
first := "b:11111111-1111-4111-8111-111111111111"
|
|
||||||
second := "b:22222222-2222-4222-8222-222222222222"
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 3)
|
|
||||||
assert.Contains(t, stats.ZfsPools, "tank")
|
|
||||||
assert.Equal(t, "ONLINE", stats.ZfsPools[first].Health)
|
|
||||||
assert.Equal(t, "DEGRADED", stats.ZfsPools[second].Health)
|
|
||||||
assert.Equal(t, uint64(100), zm.backends[1].kernelSamples[first].nread)
|
|
||||||
assert.Equal(t, uint64(200), zm.backends[1].kernelSamples[second].nread)
|
|
||||||
detail := zm.GetDetail(true)
|
|
||||||
require.Len(t, detail.Pools, 3)
|
|
||||||
assert.Equal(t, "zfs-device", detail.Pools[0].Vdevs[0].Name)
|
|
||||||
assert.Len(t, detail.Pools[0].Datasets, 1)
|
|
||||||
assert.Equal(t, "first", detail.Pools[1].Vdevs[0].Name)
|
|
||||||
assert.Empty(t, detail.Pools[1].Datasets)
|
|
||||||
assert.Equal(t, "second", detail.Pools[2].Vdevs[0].Name)
|
|
||||||
label = "renamed"
|
|
||||||
zm.backends[0].lastPoolStats = time.Time{}
|
|
||||||
zm.backends[1].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 3)
|
|
||||||
assert.Equal(t, "renamed", stats.ZfsPools[first].DisplayName)
|
|
||||||
assert.Equal(t, first, zm.GetDetail(true).Pools[1].Name)
|
|
||||||
assert.Equal(t, "renamed", zm.GetDetail(true).Pools[1].DisplayName)
|
|
||||||
}
|
|
||||||
+6
-76
@@ -4,7 +4,6 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
@@ -12,7 +11,6 @@ import (
|
|||||||
|
|
||||||
"github.com/henrygd/beszel"
|
"github.com/henrygd/beszel"
|
||||||
"github.com/henrygd/beszel/agent/battery"
|
"github.com/henrygd/beszel/agent/battery"
|
||||||
"github.com/henrygd/beszel/agent/btrfs"
|
|
||||||
"github.com/henrygd/beszel/agent/utils"
|
"github.com/henrygd/beszel/agent/utils"
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
"github.com/henrygd/beszel/internal/entities/container"
|
||||||
@@ -33,11 +31,7 @@ func (a *Agent) refreshSystemDetails() {
|
|||||||
|
|
||||||
if a.dockerManager != nil {
|
if a.dockerManager != nil {
|
||||||
a.systemDetails.Podman = a.dockerManager.IsPodman()
|
a.systemDetails.Podman = a.dockerManager.IsPodman()
|
||||||
// Docker's host info describes the machine its daemon runs on. On macOS and
|
hostInfo, _ = a.dockerManager.GetHostInfo()
|
||||||
// Windows that is a Linux VM, so its CPU and memory totals are not this host's.
|
|
||||||
if runtime.GOOS != "darwin" && runtime.GOOS != "windows" {
|
|
||||||
hostInfo, _ = a.dockerManager.GetHostInfo()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
a.systemDetails.Hostname, _ = os.Hostname()
|
a.systemDetails.Hostname, _ = os.Hostname()
|
||||||
@@ -84,12 +78,6 @@ func (a *Agent) refreshSystemDetails() {
|
|||||||
if info, err := cpu.Info(); err == nil && len(info) > 0 {
|
if info, err := cpu.Info(); err == nil && len(info) > 0 {
|
||||||
a.systemDetails.CpuModel = info[0].ModelName
|
a.systemDetails.CpuModel = info[0].ModelName
|
||||||
}
|
}
|
||||||
// gopsutil doesn't parse the "cpu model" field from /proc/cpuinfo, which
|
|
||||||
// is the only source of the CPU model name on MIPS. Fall back to reading
|
|
||||||
// it directly when ModelName is empty.
|
|
||||||
if a.systemDetails.CpuModel == "" {
|
|
||||||
a.systemDetails.CpuModel = getCpuModelFromCpuinfo()
|
|
||||||
}
|
|
||||||
// cores / threads
|
// cores / threads
|
||||||
cores, _ := cpu.Counts(false)
|
cores, _ := cpu.Counts(false)
|
||||||
threads := hostInfo.NCPU
|
threads := hostInfo.NCPU
|
||||||
@@ -176,9 +164,9 @@ func (a *Agent) getSystemStats(cacheTimeMs uint16) system.Stats {
|
|||||||
|
|
||||||
// load average
|
// load average
|
||||||
if avgstat, err := load.Avg(); err == nil {
|
if avgstat, err := load.Avg(); err == nil {
|
||||||
systemStats.LoadAvg[0] = utils.TwoDecimals(avgstat.Load1)
|
systemStats.LoadAvg[0] = avgstat.Load1
|
||||||
systemStats.LoadAvg[1] = utils.TwoDecimals(avgstat.Load5)
|
systemStats.LoadAvg[1] = avgstat.Load5
|
||||||
systemStats.LoadAvg[2] = utils.TwoDecimals(avgstat.Load15)
|
systemStats.LoadAvg[2] = avgstat.Load15
|
||||||
slog.Debug("Load average", "5m", avgstat.Load5, "15m", avgstat.Load15)
|
slog.Debug("Load average", "5m", avgstat.Load5, "15m", avgstat.Load15)
|
||||||
} else {
|
} else {
|
||||||
slog.Error("Error getting load average", "err", err)
|
slog.Error("Error getting load average", "err", err)
|
||||||
@@ -220,10 +208,6 @@ func (a *Agent) getSystemStats(cacheTimeMs uint16) system.Stats {
|
|||||||
// disk i/o (cache-aware per interval)
|
// disk i/o (cache-aware per interval)
|
||||||
a.updateDiskIo(cacheTimeMs, &systemStats)
|
a.updateDiskIo(cacheTimeMs, &systemStats)
|
||||||
|
|
||||||
// storage pool stats
|
|
||||||
a.storagePoolManager.Update(&systemStats)
|
|
||||||
a.storagePoolManager.markDuplicateCharts(&systemStats, a.fsStats, btrfs.MountID)
|
|
||||||
|
|
||||||
// network stats (per cache interval)
|
// network stats (per cache interval)
|
||||||
a.updateNetworkStats(cacheTimeMs, &systemStats)
|
a.updateNetworkStats(cacheTimeMs, &systemStats)
|
||||||
|
|
||||||
@@ -274,66 +258,13 @@ func (a *Agent) getSystemStats(cacheTimeMs uint16) system.Stats {
|
|||||||
a.systemInfo.MemPct = systemStats.MemPct
|
a.systemInfo.MemPct = systemStats.MemPct
|
||||||
a.systemInfo.DiskPct = systemStats.DiskPct
|
a.systemInfo.DiskPct = systemStats.DiskPct
|
||||||
a.systemInfo.Battery = systemStats.Battery
|
a.systemInfo.Battery = systemStats.Battery
|
||||||
a.systemInfo.Uptime, _ = getUptime()
|
a.systemInfo.Uptime, _ = host.Uptime()
|
||||||
a.systemInfo.BandwidthBytes = systemStats.Bandwidth[0] + systemStats.Bandwidth[1]
|
a.systemInfo.BandwidthBytes = systemStats.Bandwidth[0] + systemStats.Bandwidth[1]
|
||||||
a.systemInfo.Threads = a.systemDetails.Threads
|
a.systemInfo.Threads = a.systemDetails.Threads
|
||||||
|
|
||||||
return systemStats
|
return systemStats
|
||||||
}
|
}
|
||||||
|
|
||||||
// cpuModelFallbackKeys are the field names to look for in /proc/cpuinfo when
|
|
||||||
// gopsutil fails to return a ModelName. The "cpu model" key is used on MIPS
|
|
||||||
// (e.g. "MIPS 1004Kc V2.15"), while "system type" provides SoC information
|
|
||||||
// on various embedded architectures.
|
|
||||||
var cpuModelFallbackKeys = []string{"cpu model", "system type"}
|
|
||||||
|
|
||||||
// getCpuModelFromCpuinfo reads /proc/cpuinfo and returns a CPU model string.
|
|
||||||
// This is a fallback for architectures where gopsutil's cpu.Info() does not
|
|
||||||
// populate ModelName, most notably MIPS.
|
|
||||||
func getCpuModelFromCpuinfo() string {
|
|
||||||
file, err := os.Open("/proc/cpuinfo")
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
return parseCpuModel(file)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseCpuModel scans r (expected to be /proc/cpuinfo content) and returns
|
|
||||||
// a combined CPU model string. It collects values from all matching keys
|
|
||||||
// and joins them with " / " when multiple are found.
|
|
||||||
func parseCpuModel(r io.Reader) string {
|
|
||||||
lines := readLines(r)
|
|
||||||
var parts []string
|
|
||||||
for _, key := range cpuModelFallbackKeys {
|
|
||||||
for _, line := range lines {
|
|
||||||
after, found := strings.CutPrefix(line, key)
|
|
||||||
if !found {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
after = strings.TrimSpace(after)
|
|
||||||
if len(after) < 2 || after[0] != ':' {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if value := strings.TrimSpace(after[1:]); value != "" {
|
|
||||||
parts = append(parts, value)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return strings.Join(parts, " / ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// readLines reads all lines from r into a slice.
|
|
||||||
func readLines(r io.Reader) []string {
|
|
||||||
scanner := bufio.NewScanner(r)
|
|
||||||
var lines []string
|
|
||||||
for scanner.Scan() {
|
|
||||||
lines = append(lines, scanner.Text())
|
|
||||||
}
|
|
||||||
return lines
|
|
||||||
}
|
|
||||||
|
|
||||||
// calculateHostMemoryUsage derives counters defensively because /proc/meminfo may
|
// calculateHostMemoryUsage derives counters defensively because /proc/meminfo may
|
||||||
// change while gopsutil reads it. Invalid unsigned subtractions saturate at zero.
|
// change while gopsutil reads it. Invalid unsigned subtractions saturate at zero.
|
||||||
func calculateHostMemoryUsage(v *mem.VirtualMemoryStat, htop bool) (used, cacheBuff, swapUsed uint64) {
|
func calculateHostMemoryUsage(v *mem.VirtualMemoryStat, htop bool) (used, cacheBuff, swapUsed uint64) {
|
||||||
@@ -352,8 +283,7 @@ func calculateHostMemoryUsage(v *mem.VirtualMemoryStat, htop bool) (used, cacheB
|
|||||||
if htop {
|
if htop {
|
||||||
used = saturatingSub(v.Total, v.Free, cacheBuff)
|
used = saturatingSub(v.Total, v.Free, cacheBuff)
|
||||||
}
|
}
|
||||||
// Cached swap pages still occupy swap slots and are included in `free`'s used value.
|
return used, cacheBuff, saturatingSub(v.SwapTotal, v.SwapFree, v.SwapCached)
|
||||||
return used, cacheBuff, saturatingSub(v.SwapTotal, v.SwapFree)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// saturatingSub subtracts each value, returning zero on underflow.
|
// saturatingSub subtracts each value, returning zero on underflow.
|
||||||
|
|||||||
+3
-82
@@ -1,7 +1,6 @@
|
|||||||
package agent
|
package agent
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
@@ -47,14 +46,14 @@ func TestCalculateHostMemoryUsage(t *testing.T) {
|
|||||||
memory: mem.VirtualMemoryStat{Total: 100, Available: 40, Used: 60, Free: 20, Cached: 25, Buffers: 10, Shared: 5, SwapTotal: 20, SwapFree: 8, SwapCached: 2},
|
memory: mem.VirtualMemoryStat{Total: 100, Available: 40, Used: 60, Free: 20, Cached: 25, Buffers: 10, Shared: 5, SwapTotal: 20, SwapFree: 8, SwapCached: 2},
|
||||||
used: 60,
|
used: 60,
|
||||||
cacheBuff: 30,
|
cacheBuff: 30,
|
||||||
swapUsed: 12,
|
swapUsed: 10,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "inconsistent counters saturate",
|
name: "inconsistent counters saturate",
|
||||||
memory: mem.VirtualMemoryStat{Total: 100, Available: 110, Used: ^uint64(0) - 9, Free: 90, Cached: 5, Buffers: 10, Shared: 20, SwapTotal: 10, SwapFree: 9, SwapCached: 2},
|
memory: mem.VirtualMemoryStat{Total: 100, Available: 110, Used: ^uint64(0) - 9, Free: 90, Cached: 5, Buffers: 10, Shared: 20, SwapTotal: 10, SwapFree: 9, SwapCached: 2},
|
||||||
used: 0,
|
used: 0,
|
||||||
cacheBuff: 0,
|
cacheBuff: 0,
|
||||||
swapUsed: 1,
|
swapUsed: 0,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "htop subtraction saturates",
|
name: "htop subtraction saturates",
|
||||||
@@ -62,7 +61,7 @@ func TestCalculateHostMemoryUsage(t *testing.T) {
|
|||||||
htop: true,
|
htop: true,
|
||||||
used: 0,
|
used: 0,
|
||||||
cacheBuff: 25,
|
cacheBuff: 25,
|
||||||
swapUsed: 20,
|
swapUsed: 15,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "zero cache from shared cancellation does not fall back",
|
name: "zero cache from shared cancellation does not fall back",
|
||||||
@@ -114,81 +113,3 @@ func TestUpdateSystemDetailsMarksDetailsDirty(t *testing.T) {
|
|||||||
assert.False(t, agent.detailsDirty)
|
assert.False(t, agent.detailsDirty)
|
||||||
assert.Nil(t, original.Details)
|
assert.Nil(t, original.Details)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseCpuModel(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
input string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "MIPS with both cpu model and system type",
|
|
||||||
input: `system type : MediaTek MT7621 ver:1 eco:3
|
|
||||||
machine : ASUS RT-AX53U
|
|
||||||
processor : 0
|
|
||||||
cpu model : MIPS 1004Kc V2.15
|
|
||||||
BogoMIPS : 586.13
|
|
||||||
wait instruction : yes`,
|
|
||||||
expected: "MIPS 1004Kc V2.15 / MediaTek MT7621 ver:1 eco:3",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "MIPS with different SoC",
|
|
||||||
input: `system type : Atheros AR7161 rev 2
|
|
||||||
machine : NETGEAR WNDR3700
|
|
||||||
processor : 0
|
|
||||||
cpu model : MIPS 24Kc V7.4
|
|
||||||
BogoMIPS : 452.19`,
|
|
||||||
expected: "MIPS 24Kc V7.4 / Atheros AR7161 rev 2",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "only system type when cpu model missing",
|
|
||||||
input: `system type : Broadcom BCM47xx
|
|
||||||
processor : 0
|
|
||||||
BogoMIPS : 296.11`,
|
|
||||||
expected: "Broadcom BCM47xx",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "only cpu model when system type missing",
|
|
||||||
input: `processor : 0
|
|
||||||
cpu model : MIPS 34Kc V2.15
|
|
||||||
BogoMIPS : 300.00`,
|
|
||||||
expected: "MIPS 34Kc V2.15",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "x86 cpuinfo returns empty",
|
|
||||||
input: `processor : 0
|
|
||||||
vendor_id : GenuineIntel
|
|
||||||
cpu family : 6
|
|
||||||
model : 142
|
|
||||||
model name : Intel(R) Core(TM) i5-8250U CPU @ 1.60GHz
|
|
||||||
stepping : 10`,
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty input",
|
|
||||||
input: "",
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cpu model with extra whitespace",
|
|
||||||
input: `processor : 0
|
|
||||||
cpu model : MIPS 34Kc V2.15
|
|
||||||
BogoMIPS : 300.00`,
|
|
||||||
expected: "MIPS 34Kc V2.15",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cpu model without value",
|
|
||||||
input: `processor : 0
|
|
||||||
cpu model :
|
|
||||||
BogoMIPS : 300.00`,
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result := parseCpuModel(strings.NewReader(tt.input))
|
|
||||||
assert.Equal(t, tt.expected, result)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
tank 12000000000000 11999000000000 /tank
|
|
||||||
tank/apps 1000000000000 11999000000000 /tank/apps
|
|
||||||
tank/backup 2000000000000 11999000000000 /tank/backup
|
|
||||||
tank/media 1000000000000 11999000000000 /tank/my media
|
|
||||||
rpool 900000000000 300000000000 -
|
|
||||||
rpool/ROOT 1000000000 300000000000 -
|
|
||||||
rpool/ROOT/pve-1 890000000000 300000000000 /
|
|
||||||
rpool/data 9000000000 300000000000 -
|
|
||||||
rpool/data/subvol-100-disk-0 400000000000 300000000000 /subvol-100-disk-0
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
tank 23999000000000 12000000000000 11999000000000 ONLINE
|
|
||||||
rpool 1200000000000 900000000000 300000000000 DEGRADED
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
pool: tank
|
|
||||||
state: ONLINE
|
|
||||||
scan: scrub repaired 0B in 00:05:12 with 0 errors on Sun Jun 1 02:00:12 2025
|
|
||||||
config:
|
|
||||||
|
|
||||||
NAME STATE READ WRITE CKSUM
|
|
||||||
tank ONLINE 0 0 0
|
|
||||||
mirror-0 ONLINE 0 0 0
|
|
||||||
sda ONLINE 0 0 0
|
|
||||||
sdb ONLINE 0 0 0
|
|
||||||
|
|
||||||
errors: No known data errors
|
|
||||||
|
|
||||||
pool: rpool
|
|
||||||
state: DEGRADED
|
|
||||||
status: One or more devices could not be used because the label is missing or
|
|
||||||
invalid. Sufficient replicas exist for the pool to continue functioning in a
|
|
||||||
degraded state.
|
|
||||||
scan: scrub in progress since Sun Jun 8 01:00:00 2025
|
|
||||||
10.00% done, 01:30:00 to go, 0.00/s
|
|
||||||
config:
|
|
||||||
|
|
||||||
NAME STATE READ WRITE CKSUM
|
|
||||||
rpool DEGRADED 0 0 0
|
|
||||||
mirror-0 DEGRADED 0 0 0
|
|
||||||
sda ONLINE 0 0 0
|
|
||||||
sdb FAULTED 1 2 3
|
|
||||||
|
|
||||||
errors: 1 data errors, use '-v' for a list
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
//go:build linux
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/shirou/gopsutil/v4/host"
|
|
||||||
)
|
|
||||||
|
|
||||||
// uptimeFilePath is a variable so tests can point it at a fixture.
|
|
||||||
var uptimeFilePath = "/proc/uptime"
|
|
||||||
|
|
||||||
// getUptime returns the system uptime in seconds.
|
|
||||||
//
|
|
||||||
// This reads /proc/uptime instead of using host.Uptime(), which calls the
|
|
||||||
// sysinfo(2) syscall. Inside an LXC container lxcfs virtualizes /proc/uptime
|
|
||||||
// but cannot intercept a syscall, so sysinfo(2) reports the host's uptime
|
|
||||||
// rather than the container's.
|
|
||||||
//
|
|
||||||
// Falls back to host.Uptime() if /proc/uptime is missing or unparseable, so
|
|
||||||
// behavior is unchanged anywhere the file isn't available.
|
|
||||||
func getUptime() (uint64, error) {
|
|
||||||
data, err := os.ReadFile(uptimeFilePath)
|
|
||||||
if err != nil {
|
|
||||||
return host.Uptime()
|
|
||||||
}
|
|
||||||
fields := strings.Fields(string(data))
|
|
||||||
if len(fields) == 0 {
|
|
||||||
return host.Uptime()
|
|
||||||
}
|
|
||||||
seconds, err := strconv.ParseFloat(fields[0], 64)
|
|
||||||
if err != nil ||
|
|
||||||
math.IsNaN(seconds) ||
|
|
||||||
math.IsInf(seconds, 0) ||
|
|
||||||
seconds < 0 ||
|
|
||||||
seconds >= 1<<64 {
|
|
||||||
return host.Uptime()
|
|
||||||
}
|
|
||||||
return uint64(seconds), nil
|
|
||||||
}
|
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
//go:build linux
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestGetUptimeFromProc(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
contents string
|
|
||||||
want uint64
|
|
||||||
}{
|
|
||||||
{"typical", "12345.67 98765.43\n", 12345},
|
|
||||||
{"zero", "0.00 0.00\n", 0},
|
|
||||||
{"no trailing newline", "42.99 7.00", 42},
|
|
||||||
{"single field", "600.5", 600},
|
|
||||||
{"large value", "266030.12 1000000.00\n", 266030},
|
|
||||||
}
|
|
||||||
|
|
||||||
prev := uptimeFilePath
|
|
||||||
t.Cleanup(func() { uptimeFilePath = prev })
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "uptime")
|
|
||||||
if err := os.WriteFile(path, []byte(tt.contents), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
uptimeFilePath = path
|
|
||||||
|
|
||||||
got, err := getUptime()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("getUptime() returned error: %v", err)
|
|
||||||
}
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("getUptime() = %d, want %d", got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeUptime(contents string) func(t *testing.T) string {
|
|
||||||
return func(t *testing.T) string {
|
|
||||||
path := filepath.Join(t.TempDir(), "uptime")
|
|
||||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Malformed, missing, or out-of-range input must fall back to host.Uptime()
|
|
||||||
// rather than returning a bogus value, so the agent still reports something sane.
|
|
||||||
func TestGetUptimeFallsBack(t *testing.T) {
|
|
||||||
prev := uptimeFilePath
|
|
||||||
t.Cleanup(func() { uptimeFilePath = prev })
|
|
||||||
|
|
||||||
for _, tt := range []struct {
|
|
||||||
name string
|
|
||||||
prepare func(t *testing.T) string
|
|
||||||
}{
|
|
||||||
{"missing file", func(t *testing.T) string {
|
|
||||||
return filepath.Join(t.TempDir(), "does-not-exist")
|
|
||||||
}},
|
|
||||||
{"empty file", func(t *testing.T) string {
|
|
||||||
path := filepath.Join(t.TempDir(), "uptime")
|
|
||||||
if err := os.WriteFile(path, nil, 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}},
|
|
||||||
{"unparseable", func(t *testing.T) string {
|
|
||||||
path := filepath.Join(t.TempDir(), "uptime")
|
|
||||||
if err := os.WriteFile(path, []byte("not-a-number 1.0\n"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}},
|
|
||||||
{"NaN", writeUptime("NaN 1.0\n")},
|
|
||||||
{"positive infinity", writeUptime("+Inf 1.0\n")},
|
|
||||||
{"negative infinity", writeUptime("-Inf 1.0\n")},
|
|
||||||
{"negative", writeUptime("-42.5 1.0\n")},
|
|
||||||
{"exceeds uint64 range", writeUptime("1e20 1.0\n")},
|
|
||||||
} {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
uptimeFilePath = tt.prepare(t)
|
|
||||||
|
|
||||||
got, err := getUptime()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("getUptime() returned error: %v", err)
|
|
||||||
}
|
|
||||||
if got == 0 {
|
|
||||||
t.Error("getUptime() = 0, expected fallback to host.Uptime()")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
//go:build !linux
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import "github.com/shirou/gopsutil/v4/host"
|
|
||||||
|
|
||||||
// getUptime returns the system uptime in seconds.
|
|
||||||
func getUptime() (uint64, error) {
|
|
||||||
return host.Uptime()
|
|
||||||
}
|
|
||||||
@@ -1,170 +0,0 @@
|
|||||||
// Package zfs provides functions to read ZFS statistics.
|
|
||||||
package zfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var commandTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
var commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
|
|
||||||
defer cancel()
|
|
||||||
cmd := exec.CommandContext(ctx, name, args...)
|
|
||||||
cmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C")
|
|
||||||
out, err := cmd.Output()
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return nil, fmt.Errorf("%s timed out after %s: %w", name, commandTimeout, ctx.Err())
|
|
||||||
}
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrNoZfs is returned when the ZFS utilities or kernel interfaces are unavailable.
|
|
||||||
var ErrNoZfs = errors.New("zfs utilities unavailable")
|
|
||||||
|
|
||||||
// PoolStat is a snapshot of a ZFS pool's capacity and health.
|
|
||||||
type PoolStat struct {
|
|
||||||
DisplayName string // optional friendly name; Name remains the stable key
|
|
||||||
MountID string // Btrfs filesystem identity, empty for other backends
|
|
||||||
IODevice string // sole Btrfs member device, if known
|
|
||||||
Raw bool // physical accounting rather than usable filesystem space
|
|
||||||
Name string
|
|
||||||
Size uint64 // total capacity in bytes
|
|
||||||
Alloc uint64 // allocated bytes
|
|
||||||
Free uint64 // free bytes
|
|
||||||
Health string // ONLINE, DEGRADED, FAULTED, ...
|
|
||||||
}
|
|
||||||
|
|
||||||
// PoolKernelStat is the inexpensive pool telemetry exposed by the ZFS kernel.
|
|
||||||
// NRead and NWrite are cumulative byte counters since the pool was imported.
|
|
||||||
type PoolKernelStat struct {
|
|
||||||
Name string
|
|
||||||
Health string
|
|
||||||
NRead uint64
|
|
||||||
NWrite uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// PoolIoStats holds calculated per-second I/O rates for a pool.
|
|
||||||
type PoolIoStats struct {
|
|
||||||
NRead uint64
|
|
||||||
NWrite uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dataset is a single ZFS dataset with usage information.
|
|
||||||
type Dataset struct {
|
|
||||||
Name string
|
|
||||||
Used uint64
|
|
||||||
Avail uint64
|
|
||||||
Mountpoint string
|
|
||||||
}
|
|
||||||
|
|
||||||
// PoolStats returns capacity and health for all pools on the system using
|
|
||||||
// `zpool list`. Frequent health and I/O sampling uses PoolKernelStats instead.
|
|
||||||
func PoolStats() ([]PoolStat, error) {
|
|
||||||
if err := checkZfsDevice(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out, err := commandOutput("zpool", "list", "-Hp", "-o", "name,size,alloc,free,health")
|
|
||||||
if err != nil {
|
|
||||||
var exitErr *exec.ExitError
|
|
||||||
if errors.As(err, &exitErr) && strings.Contains(string(exitErr.Stderr), "no pools available") {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("zpool list: %w", err)
|
|
||||||
}
|
|
||||||
return parseZpoolListOutput(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Datasets returns all datasets on the system with usage and mountpoint
|
|
||||||
// information using `zfs list` (recursive by default).
|
|
||||||
func Datasets() ([]Dataset, error) {
|
|
||||||
if err := checkZfsDevice(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out, err := commandOutput("zfs", "list", "-Hp", "-o", "name,used,avail,mountpoint")
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("zfs list: %w", err)
|
|
||||||
}
|
|
||||||
return parseZfsListOutput(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseZpoolListOutput parses `zpool list -Hp -o name,size,alloc,free,health` output.
|
|
||||||
// Columns are tab-separated; numeric columns are raw bytes.
|
|
||||||
func parseZpoolListOutput(out []byte) ([]PoolStat, error) {
|
|
||||||
var pools []PoolStat
|
|
||||||
scanner := bufio.NewScanner(bytes.NewReader(out))
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := strings.TrimSpace(scanner.Text())
|
|
||||||
if line == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if line == "no pools available" && len(pools) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
fields := strings.Split(line, "\t")
|
|
||||||
if len(fields) < 5 {
|
|
||||||
return nil, fmt.Errorf("unexpected zpool list line: %q", line)
|
|
||||||
}
|
|
||||||
size, err := strconv.ParseUint(fields[1], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parsing size for pool %q: %w", fields[0], err)
|
|
||||||
}
|
|
||||||
alloc, err := strconv.ParseUint(fields[2], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parsing alloc for pool %q: %w", fields[0], err)
|
|
||||||
}
|
|
||||||
free, err := strconv.ParseUint(fields[3], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parsing free for pool %q: %w", fields[0], err)
|
|
||||||
}
|
|
||||||
pools = append(pools, PoolStat{
|
|
||||||
Name: fields[0],
|
|
||||||
Size: size,
|
|
||||||
Alloc: alloc,
|
|
||||||
Free: free,
|
|
||||||
Health: fields[4],
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return pools, scanner.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseZfsListOutput parses `zfs list -Hp -o name,used,avail,mountpoint` output.
|
|
||||||
// The mountpoint column may contain spaces, so it is split on tabs only.
|
|
||||||
func parseZfsListOutput(out []byte) ([]Dataset, error) {
|
|
||||||
var datasets []Dataset
|
|
||||||
scanner := bufio.NewScanner(bytes.NewReader(out))
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := strings.TrimSpace(scanner.Text())
|
|
||||||
if line == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fields := strings.SplitN(line, "\t", 4)
|
|
||||||
if len(fields) < 4 {
|
|
||||||
return nil, fmt.Errorf("unexpected zfs list line: %q", line)
|
|
||||||
}
|
|
||||||
used, err := strconv.ParseUint(fields[1], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parsing used for dataset %q: %w", fields[0], err)
|
|
||||||
}
|
|
||||||
avail, err := strconv.ParseUint(fields[2], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parsing avail for dataset %q: %w", fields[0], err)
|
|
||||||
}
|
|
||||||
datasets = append(datasets, Dataset{
|
|
||||||
Name: fields[0],
|
|
||||||
Used: used,
|
|
||||||
Avail: avail,
|
|
||||||
Mountpoint: fields[3],
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return datasets, scanner.Err()
|
|
||||||
}
|
|
||||||
@@ -3,17 +3,9 @@
|
|||||||
package zfs
|
package zfs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ARCSize() (uint64, error) {
|
func ARCSize() (uint64, error) {
|
||||||
return unix.SysctlUint64("kstat.zfs.misc.arcstats.size")
|
return unix.SysctlUint64("kstat.zfs.misc.arcstats.size")
|
||||||
}
|
}
|
||||||
|
|
||||||
// FreeBSD does not expose Linux's per-pool procfs kstats. Capacity, health,
|
|
||||||
// and detail collection still work through the cached utilities.
|
|
||||||
func PoolKernelStats() ([]PoolKernelStat, error) {
|
|
||||||
return nil, errors.ErrUnsupported
|
|
||||||
}
|
|
||||||
|
|||||||
+1
-182
@@ -5,21 +5,14 @@ package zfs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
procZfsPath = "/proc/spl/kstat/zfs"
|
|
||||||
devZfsPath = "/dev/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ARCSize() (uint64, error) {
|
func ARCSize() (uint64, error) {
|
||||||
file, err := os.Open(filepath.Join(procZfsPath, "arcstats"))
|
file, err := os.Open("/proc/spl/kstat/zfs/arcstats")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
@@ -36,180 +29,6 @@ func ARCSize() (uint64, error) {
|
|||||||
return strconv.ParseUint(fields[2], 10, 64)
|
return strconv.ParseUint(fields[2], 10, 64)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := scanner.Err(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0, fmt.Errorf("size field not found in arcstats")
|
return 0, fmt.Errorf("size field not found in arcstats")
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkZfsDevice lets containers without /dev/zfs fail fast instead of
|
|
||||||
// waiting for ZFS utility commands to time out.
|
|
||||||
func checkZfsDevice() error {
|
|
||||||
_, err := os.Stat(devZfsPath)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return ErrNoZfs
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// PoolKernelStats reads pool state and cumulative I/O counters directly from
|
|
||||||
// procfs. These kstats are the same interfaces used by node_exporter's Linux
|
|
||||||
// ZFS collector and avoid keeping a `zpool iostat` subprocess alive.
|
|
||||||
func PoolKernelStats() ([]PoolKernelStat, error) {
|
|
||||||
poolDirs := make(map[string]struct{})
|
|
||||||
for _, filename := range []string{"state", "io", "objset-*"} {
|
|
||||||
paths, err := filepath.Glob(filepath.Join(procZfsPath, "*", filename))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, path := range paths {
|
|
||||||
poolDirs[filepath.Dir(path)] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(poolDirs) == 0 {
|
|
||||||
return nil, ErrNoZfs
|
|
||||||
}
|
|
||||||
pools := make([]PoolKernelStat, 0, len(poolDirs))
|
|
||||||
for poolDir := range poolDirs {
|
|
||||||
nread, nwrite, err := readPoolCounters(poolDir)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
continue // pool may have been exported after the glob
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
state, err := os.ReadFile(filepath.Join(poolDir, "state"))
|
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
pools = append(pools, PoolKernelStat{
|
|
||||||
Name: filepath.Base(poolDir), Health: strings.ToUpper(strings.TrimSpace(string(state))),
|
|
||||||
NRead: nread, NWrite: nwrite,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if len(pools) == 0 {
|
|
||||||
return nil, ErrNoZfs
|
|
||||||
}
|
|
||||||
return pools, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readPoolCounters supports both ZFS kernel interfaces. OpenZFS through 2.3
|
|
||||||
// exposes aggregate vdev counters in "io". When that file is unavailable, sum
|
|
||||||
// the logical I/O counters exposed for each dataset in the pool.
|
|
||||||
func readPoolCounters(poolDir string) (uint64, uint64, error) {
|
|
||||||
nread, nwrite, err := readPoolIO(filepath.Join(poolDir, "io"))
|
|
||||||
if err == nil || !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nread, nwrite, err
|
|
||||||
}
|
|
||||||
return readPoolObjsets(poolDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
func readPoolIO(path string) (uint64, uint64, error) {
|
|
||||||
file, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(file)
|
|
||||||
for scanner.Scan() {
|
|
||||||
fields := strings.Fields(scanner.Text())
|
|
||||||
if len(fields) < 2 || fields[0] != "nread" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !scanner.Scan() {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
values := strings.Fields(scanner.Text())
|
|
||||||
if len(values) < 2 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
nread, err := strconv.ParseUint(values[0], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf("parsing nread in %s: %w", path, err)
|
|
||||||
}
|
|
||||||
nwrite, err := strconv.ParseUint(values[1], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf("parsing nwritten in %s: %w", path, err)
|
|
||||||
}
|
|
||||||
return nread, nwrite, nil
|
|
||||||
}
|
|
||||||
if err := scanner.Err(); err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
return 0, 0, fmt.Errorf("I/O counters not found in %s", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func readPoolObjsets(poolDir string) (uint64, uint64, error) {
|
|
||||||
paths, err := filepath.Glob(filepath.Join(poolDir, "objset-*"))
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
if len(paths) == 0 {
|
|
||||||
return 0, 0, fmt.Errorf("dataset I/O counters not found in %s", poolDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
var totalRead, totalWrite uint64
|
|
||||||
objsetsRead := 0
|
|
||||||
for _, path := range paths {
|
|
||||||
nread, nwrite, err := readObjsetIO(path)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
continue // dataset may have been destroyed after the glob
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
totalRead += nread
|
|
||||||
totalWrite += nwrite
|
|
||||||
objsetsRead++
|
|
||||||
}
|
|
||||||
if objsetsRead == 0 {
|
|
||||||
return 0, 0, fmt.Errorf("dataset I/O counters not found in %s", poolDir)
|
|
||||||
}
|
|
||||||
return totalRead, totalWrite, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func readObjsetIO(path string) (uint64, uint64, error) {
|
|
||||||
file, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
var nread, nwrite uint64
|
|
||||||
var foundRead, foundWrite bool
|
|
||||||
scanner := bufio.NewScanner(file)
|
|
||||||
for scanner.Scan() {
|
|
||||||
fields := strings.Fields(scanner.Text())
|
|
||||||
if len(fields) < 3 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var target *uint64
|
|
||||||
switch fields[0] {
|
|
||||||
case "nread":
|
|
||||||
target = &nread
|
|
||||||
foundRead = true
|
|
||||||
case "nwritten":
|
|
||||||
target = &nwrite
|
|
||||||
foundWrite = true
|
|
||||||
default:
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
value, err := strconv.ParseUint(fields[2], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf("parsing %s in %s: %w", fields[0], path, err)
|
|
||||||
}
|
|
||||||
*target = value
|
|
||||||
}
|
|
||||||
if err := scanner.Err(); err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
if !foundRead || !foundWrite {
|
|
||||||
return 0, 0, fmt.Errorf("incomplete I/O counters in %s", path)
|
|
||||||
}
|
|
||||||
return nread, nwrite, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,153 +0,0 @@
|
|||||||
//go:build testing && linux
|
|
||||||
|
|
||||||
package zfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPoolKernelStats(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldPath := procZfsPath
|
|
||||||
procZfsPath = root
|
|
||||||
t.Cleanup(func() { procZfsPath = oldPath })
|
|
||||||
|
|
||||||
poolDir := filepath.Join(root, "tank")
|
|
||||||
require.NoError(t, os.MkdirAll(poolDir, 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "io"), []byte(
|
|
||||||
"11 3 0x00 1 80 0 0\n"+
|
|
||||||
"nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt\n"+
|
|
||||||
"1884160 6450688 22 978 0 0 0 0 0 0 0 0\n",
|
|
||||||
), 0o644))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "state"), []byte("DEGRADED\n"), 0o644))
|
|
||||||
|
|
||||||
stats, err := PoolKernelStats()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, stats, 1)
|
|
||||||
assert.Equal(t, PoolKernelStat{
|
|
||||||
Name: "tank", Health: "DEGRADED", NRead: 1884160, NWrite: 6450688,
|
|
||||||
}, stats[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPoolKernelStatsOpenZfs24(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldPath := procZfsPath
|
|
||||||
procZfsPath = root
|
|
||||||
t.Cleanup(func() { procZfsPath = oldPath })
|
|
||||||
|
|
||||||
poolDir := filepath.Join(root, "tank")
|
|
||||||
require.NoError(t, os.MkdirAll(poolDir, 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "state"), []byte("ONLINE\n"), 0o644))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "objset-0x1"), []byte(
|
|
||||||
"34 1 0x01 28 7872 0 0\n"+
|
|
||||||
"name type data\n"+
|
|
||||||
"dataset_name 7 tank\n"+
|
|
||||||
"nwritten 4 2000\n"+
|
|
||||||
"nread 4 1000\n",
|
|
||||||
), 0o644))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "objset-0x2"), []byte(
|
|
||||||
"34 1 0x01 28 7872 0 0\n"+
|
|
||||||
"name type data\n"+
|
|
||||||
"dataset_name 7 tank/videos\n"+
|
|
||||||
"nwritten 4 400\n"+
|
|
||||||
"nread 4 300\n",
|
|
||||||
), 0o644))
|
|
||||||
|
|
||||||
stats, err := PoolKernelStats()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, stats, 1)
|
|
||||||
assert.Equal(t, PoolKernelStat{
|
|
||||||
Name: "tank", Health: "ONLINE", NRead: 1300, NWrite: 2400,
|
|
||||||
}, stats[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPoolKernelStatsNoZfs(t *testing.T) {
|
|
||||||
oldPath := procZfsPath
|
|
||||||
procZfsPath = t.TempDir()
|
|
||||||
t.Cleanup(func() { procZfsPath = oldPath })
|
|
||||||
|
|
||||||
_, err := PoolKernelStats()
|
|
||||||
assert.ErrorIs(t, err, ErrNoZfs)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadPoolIORejectsMalformedCounters(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "io")
|
|
||||||
require.NoError(t, os.WriteFile(path, []byte("nread nwritten\nnope 10\n"), 0o644))
|
|
||||||
_, _, err := readPoolIO(path)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadObjsetIORequiresAllCounters(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "objset-0x1")
|
|
||||||
require.NoError(t, os.WriteFile(path, []byte("nread 4 10\n"), 0o644))
|
|
||||||
_, _, err := readObjsetIO(path)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCollectorsSkipCommandsWhenDevZfsMissing(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldDevZfsPath := devZfsPath
|
|
||||||
devZfsPath = filepath.Join(root, "missing")
|
|
||||||
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
|
|
||||||
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
t.Fatalf("unexpected %s call with %v", name, args)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
_, err := PoolStats()
|
|
||||||
assert.ErrorIs(t, err, ErrNoZfs)
|
|
||||||
_, err = Datasets()
|
|
||||||
assert.ErrorIs(t, err, ErrNoZfs)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetsDelegatesWhenDevZfsPresent(t *testing.T) {
|
|
||||||
oldDevZfsPath := devZfsPath
|
|
||||||
devZfsPath = filepath.Join(t.TempDir(), "zfs")
|
|
||||||
require.NoError(t, os.WriteFile(devZfsPath, nil, 0o644))
|
|
||||||
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
|
|
||||||
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
assert.Equal(t, "zfs", name)
|
|
||||||
assert.Equal(t, []string{"list", "-Hp", "-o", "name,used,avail,mountpoint"}, args)
|
|
||||||
return []byte("tank\t50\t50\t/tank\n"), nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
datasets, err := Datasets()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []Dataset{{Name: "tank", Used: 50, Avail: 50, Mountpoint: "/tank"}}, datasets)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPoolStatsDelegatesToZpoolWhenDevZfsPresent(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
devFile := filepath.Join(root, "zfs")
|
|
||||||
require.NoError(t, os.WriteFile(devFile, []byte(""), 0o644))
|
|
||||||
|
|
||||||
oldDevZfsPath := devZfsPath
|
|
||||||
devZfsPath = devFile
|
|
||||||
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
|
|
||||||
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
called := false
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
called = true
|
|
||||||
assert.Equal(t, "zpool", name)
|
|
||||||
assert.Equal(t, []string{"list", "-Hp", "-o", "name,size,alloc,free,health"}, args)
|
|
||||||
return []byte("tank\t100\t50\t50\tONLINE\n"), nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
pools, err := PoolStats()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, called)
|
|
||||||
assert.Equal(t, []PoolStat{{Name: "tank", Size: 100, Alloc: 50, Free: 50, Health: "ONLINE"}}, pools)
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
//go:build !linux
|
|
||||||
|
|
||||||
package zfs
|
|
||||||
|
|
||||||
// The /dev/zfs probe is Linux-specific. Other platforms detect availability
|
|
||||||
// through the ZFS utilities themselves.
|
|
||||||
func checkZfsDevice() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
//go:build testing && !linux
|
|
||||||
|
|
||||||
package zfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCollectorsUseUtilitiesOnNonLinux(t *testing.T) {
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
switch name {
|
|
||||||
case "zpool":
|
|
||||||
return []byte("tank\t100\t50\t50\tONLINE\n"), nil
|
|
||||||
case "zfs":
|
|
||||||
return []byte("tank\t50\t50\t/tank\n"), nil
|
|
||||||
default:
|
|
||||||
t.Fatalf("unexpected command %s", name)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
pools, err := PoolStats()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []PoolStat{{Name: "tank", Size: 100, Alloc: 50, Free: 50, Health: "ONLINE"}}, pools)
|
|
||||||
datasets, err := Datasets()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []Dataset{{Name: "tank", Used: 50, Avail: 50, Mountpoint: "/tank"}}, datasets)
|
|
||||||
}
|
|
||||||
@@ -1,150 +0,0 @@
|
|||||||
package zfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// PoolStatus holds parsed `zpool status` information for one pool.
|
|
||||||
type PoolStatus struct {
|
|
||||||
Name string
|
|
||||||
State string // ONLINE, DEGRADED, FAULTED, ...
|
|
||||||
Scrub ScrubStatus
|
|
||||||
Vdevs []VdevStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
// ScrubStatus holds the scrub (or resilver) status parsed from the scan line.
|
|
||||||
type ScrubStatus struct {
|
|
||||||
State string // NONE, SCANNING, FINISHED, CANCELED
|
|
||||||
Progress string // e.g. "10.00%" while scanning
|
|
||||||
Errors uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// VdevStatus is a single vdev row (mirror, raidz, or leaf disk).
|
|
||||||
type VdevStatus struct {
|
|
||||||
Name string
|
|
||||||
State string
|
|
||||||
ReadErrs uint64
|
|
||||||
WriteErrs uint64
|
|
||||||
ChecksumErrs uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
progressRe = regexp.MustCompile(`(\d+\.\d+)%\s+done`)
|
|
||||||
errorsRe = regexp.MustCompile(`with\s+(\d+)\s+errors`)
|
|
||||||
)
|
|
||||||
|
|
||||||
// PoolStatuses runs `zpool status` and parses per-pool state, scrub, and vdev
|
|
||||||
// information. The human-readable format has been stable across OpenZFS
|
|
||||||
// releases; rows are matched by their tabular shape rather than position.
|
|
||||||
func PoolStatuses() ([]PoolStatus, error) {
|
|
||||||
out, err := commandOutput("zpool", "status")
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("zpool status: %w", err)
|
|
||||||
}
|
|
||||||
return parseZpoolStatusOutput(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseZpoolStatusOutput parses the output of `zpool status`.
|
|
||||||
func parseZpoolStatusOutput(out []byte) ([]PoolStatus, error) {
|
|
||||||
var pools []PoolStatus
|
|
||||||
var current *PoolStatus
|
|
||||||
inConfig := false
|
|
||||||
scanContinuation := false // next non-blank line continues the scan line (progress)
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(bytes.NewReader(out))
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := scanner.Text()
|
|
||||||
trimmed := strings.TrimSpace(line)
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(trimmed, "pool:"):
|
|
||||||
pools = append(pools, PoolStatus{Name: strings.TrimSpace(strings.TrimPrefix(trimmed, "pool:"))})
|
|
||||||
current = &pools[len(pools)-1]
|
|
||||||
inConfig = false
|
|
||||||
scanContinuation = false
|
|
||||||
case current == nil:
|
|
||||||
continue
|
|
||||||
case strings.HasPrefix(trimmed, "state:"):
|
|
||||||
current.State = strings.TrimSpace(strings.TrimPrefix(trimmed, "state:"))
|
|
||||||
case strings.HasPrefix(trimmed, "scan:"):
|
|
||||||
current.Scrub = parseScanLine(trimmed)
|
|
||||||
// zpool status prints the progress percentage on the line after scan.
|
|
||||||
scanContinuation = true
|
|
||||||
case trimmed == "config:":
|
|
||||||
inConfig = true
|
|
||||||
case scanContinuation:
|
|
||||||
// The line after scan: may be an indented progress continuation.
|
|
||||||
if m := progressRe.FindStringSubmatch(trimmed); m != nil {
|
|
||||||
current.Scrub.Progress = m[1] + "%"
|
|
||||||
}
|
|
||||||
scanContinuation = false
|
|
||||||
case inConfig && (line == "" || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")):
|
|
||||||
// Table rows are indented; blank lines separate sections. The
|
|
||||||
// column header and the pool's own row are skipped.
|
|
||||||
if trimmed != "" && !strings.HasPrefix(trimmed, "NAME") {
|
|
||||||
if vdev, ok := parseVdevLine(trimmed, current.Name); ok {
|
|
||||||
current.Vdevs = append(current.Vdevs, vdev)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case inConfig:
|
|
||||||
// unindented line (errors:, status:, next pool:) ends the table
|
|
||||||
inConfig = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return pools, scanner.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseScanLine maps a `scan:` line to a ScrubStatus.
|
|
||||||
func parseScanLine(line string) ScrubStatus {
|
|
||||||
var scrub ScrubStatus
|
|
||||||
switch {
|
|
||||||
case strings.Contains(line, "in progress"):
|
|
||||||
scrub.State = "SCANNING"
|
|
||||||
case strings.Contains(line, "canceled"):
|
|
||||||
scrub.State = "CANCELED"
|
|
||||||
case strings.Contains(line, "repaired"), strings.Contains(line, "resilvered"):
|
|
||||||
scrub.State = "FINISHED"
|
|
||||||
default:
|
|
||||||
scrub.State = "NONE"
|
|
||||||
}
|
|
||||||
if m := progressRe.FindStringSubmatch(line); m != nil {
|
|
||||||
scrub.Progress = m[1] + "%"
|
|
||||||
}
|
|
||||||
if m := errorsRe.FindStringSubmatch(line); m != nil {
|
|
||||||
if n, err := strconv.ParseUint(m[1], 10, 64); err == nil {
|
|
||||||
scrub.Errors = n
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return scrub
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseVdevLine parses one row of the config table. Rows have the shape
|
|
||||||
// "NAME STATE READ WRITE CKSUM [extra...]". The first data row is the pool
|
|
||||||
// itself and is skipped since it duplicates pool-level info.
|
|
||||||
func parseVdevLine(line, poolName string) (VdevStatus, bool) {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) < 5 {
|
|
||||||
return VdevStatus{}, false
|
|
||||||
}
|
|
||||||
if fields[0] == poolName {
|
|
||||||
return VdevStatus{}, false
|
|
||||||
}
|
|
||||||
read, err1 := strconv.ParseUint(fields[2], 10, 64)
|
|
||||||
write, err2 := strconv.ParseUint(fields[3], 10, 64)
|
|
||||||
cksum, err3 := strconv.ParseUint(fields[4], 10, 64)
|
|
||||||
if err1 != nil || err2 != nil || err3 != nil {
|
|
||||||
return VdevStatus{}, false
|
|
||||||
}
|
|
||||||
return VdevStatus{
|
|
||||||
Name: fields[0],
|
|
||||||
State: fields[1],
|
|
||||||
ReadErrs: read,
|
|
||||||
WriteErrs: write,
|
|
||||||
ChecksumErrs: cksum,
|
|
||||||
}, true
|
|
||||||
}
|
|
||||||
@@ -1,143 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package zfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func fixturePath(name string) string {
|
|
||||||
return filepath.Join("..", "test-data", "zfs", name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseZpoolListOutput(t *testing.T) {
|
|
||||||
data, err := os.ReadFile(fixturePath("zpool_list.txt"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
pools, err := parseZpoolListOutput(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, pools, 2)
|
|
||||||
|
|
||||||
assert.Equal(t, PoolStat{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "ONLINE"}, pools[0])
|
|
||||||
assert.Equal(t, PoolStat{Name: "rpool", Size: 1200000000000, Alloc: 900000000000, Free: 300000000000, Health: "DEGRADED"}, pools[1])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseZpoolListOutputIgnoresEmptyLines(t *testing.T) {
|
|
||||||
pools, err := parseZpoolListOutput([]byte("tank\t100\t50\t50\tONLINE\n\n"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, pools, 1)
|
|
||||||
assert.Equal(t, "tank", pools[0].Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseZpoolListOutputNoPools(t *testing.T) {
|
|
||||||
pools, err := parseZpoolListOutput([]byte("no pools available\n"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, pools)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseZpoolListOutputRejectsMalformedLine(t *testing.T) {
|
|
||||||
_, err := parseZpoolListOutput([]byte("tank\t100\t50\n"))
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
_, err = parseZpoolListOutput([]byte("tank\tnotanumber\t50\t50\tONLINE\n"))
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseZfsListOutput(t *testing.T) {
|
|
||||||
data, err := os.ReadFile(fixturePath("zfs_list.txt"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
datasets, err := parseZfsListOutput(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, datasets, 9)
|
|
||||||
|
|
||||||
// Mountpoint with a space must be kept intact (tab-split only).
|
|
||||||
assert.Equal(t, "/tank/my media", datasets[3].Mountpoint)
|
|
||||||
// Unmounted datasets/zvols report "-".
|
|
||||||
assert.Equal(t, "-", datasets[4].Mountpoint)
|
|
||||||
assert.Equal(t, uint64(12000000000000), datasets[0].Used)
|
|
||||||
assert.Equal(t, uint64(11999000000000), datasets[0].Avail)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseZpoolStatusOutput(t *testing.T) {
|
|
||||||
data, err := os.ReadFile(fixturePath("zpool_status.txt"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
pools, err := parseZpoolStatusOutput(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, pools, 2)
|
|
||||||
|
|
||||||
tank := pools[0]
|
|
||||||
assert.Equal(t, "tank", tank.Name)
|
|
||||||
assert.Equal(t, "ONLINE", tank.State)
|
|
||||||
assert.Equal(t, "FINISHED", tank.Scrub.State)
|
|
||||||
assert.Equal(t, "", tank.Scrub.Progress)
|
|
||||||
assert.Equal(t, uint64(0), tank.Scrub.Errors)
|
|
||||||
// Pool row itself is skipped; mirror + 2 disks remain.
|
|
||||||
require.Len(t, tank.Vdevs, 3)
|
|
||||||
assert.Equal(t, "mirror-0", tank.Vdevs[0].Name)
|
|
||||||
assert.Equal(t, "sda", tank.Vdevs[1].Name)
|
|
||||||
assert.Equal(t, "sdb", tank.Vdevs[2].Name)
|
|
||||||
|
|
||||||
rpool := pools[1]
|
|
||||||
assert.Equal(t, "rpool", rpool.Name)
|
|
||||||
assert.Equal(t, "DEGRADED", rpool.State)
|
|
||||||
assert.Equal(t, "SCANNING", rpool.Scrub.State)
|
|
||||||
assert.Equal(t, "10.00%", rpool.Scrub.Progress)
|
|
||||||
require.Len(t, rpool.Vdevs, 3)
|
|
||||||
assert.Equal(t, "FAULTED", rpool.Vdevs[2].State)
|
|
||||||
assert.Equal(t, uint64(1), rpool.Vdevs[2].ReadErrs)
|
|
||||||
assert.Equal(t, uint64(2), rpool.Vdevs[2].WriteErrs)
|
|
||||||
assert.Equal(t, uint64(3), rpool.Vdevs[2].ChecksumErrs)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseScanLine(t *testing.T) {
|
|
||||||
assert.Equal(t, "FINISHED", parseScanLine("scan: scrub repaired 0B in 00:05:12 with 0 errors on Sun Jun 1 02:00:12 2025").State)
|
|
||||||
assert.Equal(t, uint64(3), parseScanLine("scan: scrub repaired 10G in 01:00:00 with 3 errors on Sun Jun 1 02:00:12 2025").Errors)
|
|
||||||
assert.Equal(t, "SCANNING", parseScanLine("scan: scrub in progress since Sun Jun 8 01:00:00 2025").State)
|
|
||||||
assert.Equal(t, "CANCELED", parseScanLine("scan: scrub canceled on Sun Jun 1 02:00:12 2025").State)
|
|
||||||
assert.Equal(t, "FINISHED", parseScanLine("scan: resilvered 1.23G in 00:01:00 with 0 errors on Sun Jun 1 02:00:12 2025").State)
|
|
||||||
assert.Equal(t, "NONE", parseScanLine("scan: none requested").State)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCommandOutputForcesLocaleAndTimesOut(t *testing.T) {
|
|
||||||
t.Setenv("BESZEL_ZFS_COMMAND_HELPER", "1")
|
|
||||||
out, err := commandOutput(os.Args[0], "-test.run=TestZfsCommandHelperProcess", "--", "locale")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "C/C", string(out))
|
|
||||||
|
|
||||||
oldTimeout := commandTimeout
|
|
||||||
commandTimeout = 20 * time.Millisecond
|
|
||||||
t.Cleanup(func() { commandTimeout = oldTimeout })
|
|
||||||
_, err = commandOutput(os.Args[0], "-test.run=TestZfsCommandHelperProcess", "--", "sleep")
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "timed out")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestZfsCommandHelperProcess(t *testing.T) {
|
|
||||||
if os.Getenv("BESZEL_ZFS_COMMAND_HELPER") != "1" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
mode := ""
|
|
||||||
for i, arg := range os.Args {
|
|
||||||
if arg == "--" && i+1 < len(os.Args) {
|
|
||||||
mode = os.Args[i+1]
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch strings.TrimSpace(mode) {
|
|
||||||
case "locale":
|
|
||||||
_, _ = fmt.Printf("%s/%s", os.Getenv("LC_ALL"), os.Getenv("LANG"))
|
|
||||||
case "sleep":
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
}
|
|
||||||
os.Exit(0)
|
|
||||||
}
|
|
||||||
@@ -7,7 +7,3 @@ import "errors"
|
|||||||
func ARCSize() (uint64, error) {
|
func ARCSize() (uint64, error) {
|
||||||
return 0, errors.ErrUnsupported
|
return 0, errors.ErrUnsupported
|
||||||
}
|
}
|
||||||
|
|
||||||
func PoolKernelStats() ([]PoolKernelStat, error) {
|
|
||||||
return nil, errors.ErrUnsupported
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// Version is the current version of the application.
|
// Version is the current version of the application.
|
||||||
Version = "0.20.0"
|
Version = "0.18.7"
|
||||||
// AppName is the name of the application.
|
// AppName is the name of the application.
|
||||||
AppName = "beszel"
|
AppName = "beszel"
|
||||||
)
|
)
|
||||||
@@ -16,9 +16,3 @@ var MinVersionCbor = semver.MustParse("0.12.0")
|
|||||||
|
|
||||||
// MinVersionAgentResponse is the minimum supported version for AgentResponse compatibility.
|
// MinVersionAgentResponse is the minimum supported version for AgentResponse compatibility.
|
||||||
var MinVersionAgentResponse = semver.MustParse("0.13.0")
|
var MinVersionAgentResponse = semver.MustParse("0.13.0")
|
||||||
|
|
||||||
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
|
|
||||||
var MinVersionZfsData = semver.MustParse("0.18.9")
|
|
||||||
|
|
||||||
// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.
|
|
||||||
var MinVersionNetworkMonitors = semver.MustParse("0.20.0")
|
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
entries: {}
|
||||||
|
generated: "2026-03-05T00:00:00Z"
|
||||||
@@ -1,27 +1,26 @@
|
|||||||
module github.com/henrygd/beszel
|
module github.com/henrygd/beszel
|
||||||
|
|
||||||
go 1.27.1
|
go 1.26.6
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/blang/semver v3.5.1+incompatible
|
github.com/blang/semver v3.5.1+incompatible
|
||||||
github.com/coreos/go-systemd/v22 v22.7.0
|
github.com/coreos/go-systemd/v22 v22.7.0
|
||||||
github.com/distribution/reference v0.6.0
|
github.com/ebitengine/purego v0.10.2
|
||||||
github.com/ebitengine/purego v0.11.0
|
github.com/fxamacker/cbor/v2 v2.9.2
|
||||||
github.com/fxamacker/cbor/v2 v2.9.3
|
|
||||||
github.com/gliderlabs/ssh v0.3.8
|
github.com/gliderlabs/ssh v0.3.8
|
||||||
|
github.com/google/uuid v1.6.0
|
||||||
github.com/lxzan/gws v1.10.1
|
github.com/lxzan/gws v1.10.1
|
||||||
github.com/nicholas-fedor/shoutrrr v0.20.0
|
github.com/nicholas-fedor/shoutrrr v0.17.0
|
||||||
github.com/opencontainers/go-digest v1.0.0
|
|
||||||
github.com/pocketbase/dbx v1.12.0
|
github.com/pocketbase/dbx v1.12.0
|
||||||
github.com/pocketbase/pocketbase v0.40.2
|
github.com/pocketbase/pocketbase v0.39.10
|
||||||
github.com/shirou/gopsutil/v4 v4.26.8
|
github.com/shirou/gopsutil/v4 v4.26.7
|
||||||
github.com/spf13/cast v1.10.0
|
github.com/spf13/cast v1.10.0
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
github.com/spf13/pflag v1.0.10
|
github.com/spf13/pflag v1.0.10
|
||||||
github.com/stretchr/testify v1.12.1
|
github.com/stretchr/testify v1.11.1
|
||||||
golang.org/x/crypto v0.56.0
|
golang.org/x/crypto v0.55.0
|
||||||
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa
|
golang.org/x/exp v0.0.0-20260811152304-ee035b5b010f
|
||||||
golang.org/x/net v0.58.0
|
golang.org/x/net v0.57.0
|
||||||
golang.org/x/sys v0.47.0
|
golang.org/x/sys v0.47.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
howett.net/plist v1.0.1
|
howett.net/plist v1.0.1
|
||||||
@@ -30,6 +29,7 @@ require (
|
|||||||
require (
|
require (
|
||||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
|
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
|
||||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/disintegration/imaging v1.6.2 // indirect
|
github.com/disintegration/imaging v1.6.2 // indirect
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
|
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
@@ -42,14 +42,14 @@ require (
|
|||||||
github.com/go-sql-driver/mysql v1.9.1 // indirect
|
github.com/go-sql-driver/mysql v1.9.1 // indirect
|
||||||
github.com/godbus/dbus/v5 v5.2.2 // indirect
|
github.com/godbus/dbus/v5 v5.2.2 // indirect
|
||||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
|
||||||
github.com/gorilla/websocket v1.5.3 // indirect
|
github.com/gorilla/websocket v1.5.3 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/klauspost/compress v1.20.0 // indirect
|
github.com/klauspost/compress v1.19.2 // indirect
|
||||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 // indirect
|
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 // indirect
|
||||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
|
||||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
|
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
@@ -57,14 +57,13 @@ require (
|
|||||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||||
github.com/x448/float16 v0.8.4 // indirect
|
github.com/x448/float16 v0.8.4 // indirect
|
||||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
|
||||||
golang.org/x/image v0.45.0 // indirect
|
golang.org/x/image v0.45.0 // indirect
|
||||||
golang.org/x/oauth2 v0.36.0 // indirect
|
golang.org/x/oauth2 v0.36.0 // indirect
|
||||||
golang.org/x/sync v0.22.0 // indirect
|
golang.org/x/sync v0.22.0 // indirect
|
||||||
golang.org/x/term v0.45.0 // indirect
|
golang.org/x/term v0.45.0 // indirect
|
||||||
golang.org/x/text v0.41.0 // indirect
|
golang.org/x/text v0.41.0 // indirect
|
||||||
modernc.org/libc v1.74.4 // indirect
|
modernc.org/libc v1.74.1 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.12.1 // indirect
|
modernc.org/memory v1.12.0 // indirect
|
||||||
modernc.org/sqlite v1.57.0 // indirect
|
modernc.org/sqlite v1.55.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -13,16 +13,16 @@ github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj
|
|||||||
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
|
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
||||||
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
||||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
|
||||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
|
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
|
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
|
github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE=
|
||||||
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||||
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
|
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
|
||||||
github.com/eclipse/paho.golang v0.23.0/go.mod h1:nQRhTkoZv8EAiNs5UU0/WdQIx2NrnWUpL9nsGJTQN04=
|
github.com/eclipse/paho.golang v0.23.0/go.mod h1:nQRhTkoZv8EAiNs5UU0/WdQIx2NrnWUpL9nsGJTQN04=
|
||||||
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
|
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
|
||||||
@@ -31,8 +31,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
|||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng3Q=
|
github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78=
|
||||||
github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
||||||
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
|
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
|
||||||
@@ -56,8 +56,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArs
|
|||||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe h1:QAinXoAFJdGQYztXn3VpFey7KCwpedbZ/EkzbplQ0cY=
|
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||||
github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||||
@@ -69,8 +69,8 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
|
|||||||
github.com/jarcoal/httpmock v1.4.2 h1:dKwiP/9zITCPfBLsDn3kchbSOu16JrnxtVEmL0fPRcI=
|
github.com/jarcoal/httpmock v1.4.2 h1:dKwiP/9zITCPfBLsDn3kchbSOu16JrnxtVEmL0fPRcI=
|
||||||
github.com/jarcoal/httpmock v1.4.2/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0=
|
github.com/jarcoal/httpmock v1.4.2/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0=
|
||||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
@@ -85,21 +85,21 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe
|
|||||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||||
github.com/nicholas-fedor/shoutrrr v0.20.0 h1:hMAxIYlfAeZ1FcTDgU0kUOvVXUsOirWo8IWlnzGLkac=
|
github.com/nicholas-fedor/shoutrrr v0.17.0 h1:xfp3z5QbE8jXvUhUEwWDk47SJ/b912VoB8MJJDU+q4E=
|
||||||
github.com/nicholas-fedor/shoutrrr v0.20.0/go.mod h1:hgde37yNWCXh8+N6WemyDRMNYLOFTf326GsBx8Z7CFA=
|
github.com/nicholas-fedor/shoutrrr v0.17.0/go.mod h1:s4ldyLs6uwBy9lIjYrY+8lyTqJtPvZSrILw0CyMLock=
|
||||||
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw=
|
github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E=
|
||||||
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
||||||
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
|
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
|
||||||
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
|
||||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA=
|
github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA=
|
||||||
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
|
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
|
||||||
github.com/pocketbase/pocketbase v0.40.2 h1:7gTqvt3bmilkphyZZ1QNhX19g3BXHqT7ynDyU81RVT4=
|
github.com/pocketbase/pocketbase v0.39.10 h1:2j8TDJRuo3aAC8Y8F9WFux0SwYcxeDCgEYQxxdWkwGE=
|
||||||
github.com/pocketbase/pocketbase v0.40.2/go.mod h1:jc3YuyToy+ZXM4CeO7uSCN/htgR8yv+tjSE3eJZ8eh8=
|
github.com/pocketbase/pocketbase v0.39.10/go.mod h1:tSX3anHQ7Ul6dPV9WhlEc6No1DtklGF69iwnVNW3BEE=
|
||||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
|
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
|
||||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
@@ -107,8 +107,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq
|
|||||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo=
|
github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc=
|
||||||
github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM=
|
github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM=
|
||||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||||
@@ -120,8 +120,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||||
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||||
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||||
@@ -136,18 +136,18 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
|||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||||
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||||
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
|
golang.org/x/exp v0.0.0-20260811152304-ee035b5b010f h1:iXpLj9sdDH/RLYsnOMpbETK6KWtrHwvegcc4psWJHV8=
|
||||||
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
|
golang.org/x/exp v0.0.0-20260811152304-ee035b5b010f/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
|
||||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||||
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
||||||
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
||||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
@@ -165,19 +165,20 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
|||||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||||
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
|
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
|
||||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
|
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
|
||||||
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
||||||
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
|
modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
|
||||||
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||||
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
|
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
|
||||||
modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
|
modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
|
||||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||||
@@ -188,18 +189,18 @@ modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
|
|||||||
modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||||
modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
|
modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ=
|
||||||
modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
|
modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os=
|
||||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||||
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
modernc.org/memory v1.12.0 h1:twkmYNkGXCvtYWzoux02jtK6eovjZbdI0uHFUYp6kuU=
|
||||||
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
modernc.org/memory v1.12.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||||
modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg=
|
modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM=
|
||||||
modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
|
modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw=
|
||||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||||
|
|||||||
@@ -20,11 +20,10 @@ type hubLike interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type AlertManager struct {
|
type AlertManager struct {
|
||||||
hub hubLike
|
hub hubLike
|
||||||
stopOnce sync.Once
|
stopOnce sync.Once
|
||||||
pendingAlerts sync.Map
|
pendingAlerts sync.Map
|
||||||
alertsCache *AlertsCache
|
alertsCache *AlertsCache
|
||||||
networkMonitors *networkMonitorCache
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type AlertMessageData struct {
|
type AlertMessageData struct {
|
||||||
@@ -49,7 +48,6 @@ type SystemAlertFsStats struct {
|
|||||||
// Values pulled from system_stats.stats that are relevant to alerts.
|
// Values pulled from system_stats.stats that are relevant to alerts.
|
||||||
type SystemAlertStats struct {
|
type SystemAlertStats struct {
|
||||||
Cpu float64 `json:"cpu"`
|
Cpu float64 `json:"cpu"`
|
||||||
CpuBreakdown []float64 `json:"cpub"`
|
|
||||||
Mem float64 `json:"mp"`
|
Mem float64 `json:"mp"`
|
||||||
Disk float64 `json:"dp"`
|
Disk float64 `json:"dp"`
|
||||||
Bandwidth [2]uint64 `json:"b"`
|
Bandwidth [2]uint64 `json:"b"`
|
||||||
@@ -59,19 +57,12 @@ type SystemAlertStats struct {
|
|||||||
Battery [2]uint8 `json:"bat"`
|
Battery [2]uint8 `json:"bat"`
|
||||||
Batteries map[string]uint8 `json:"bats"`
|
Batteries map[string]uint8 `json:"bats"`
|
||||||
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
|
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
|
||||||
ZfsPools map[string]SystemAlertZfsPool `json:"z"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type SystemAlertGPUData struct {
|
type SystemAlertGPUData struct {
|
||||||
Usage float64 `json:"u"`
|
Usage float64 `json:"u"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SystemAlertZfsPool struct {
|
|
||||||
Raw bool `json:"raw,omitempty"`
|
|
||||||
Total float64 `json:"d"`
|
|
||||||
Used float64 `json:"du"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SystemAlertData struct {
|
type SystemAlertData struct {
|
||||||
systemRecord *core.Record
|
systemRecord *core.Record
|
||||||
alertData CachedAlertData
|
alertData CachedAlertData
|
||||||
@@ -108,9 +99,8 @@ var supportsTitle = map[string]struct{}{
|
|||||||
// NewAlertManager creates a new AlertManager instance.
|
// NewAlertManager creates a new AlertManager instance.
|
||||||
func NewAlertManager(app hubLike) *AlertManager {
|
func NewAlertManager(app hubLike) *AlertManager {
|
||||||
am := &AlertManager{
|
am := &AlertManager{
|
||||||
hub: app,
|
hub: app,
|
||||||
alertsCache: NewAlertsCache(app),
|
alertsCache: NewAlertsCache(app),
|
||||||
networkMonitors: newNetworkMonitorCache(app),
|
|
||||||
}
|
}
|
||||||
am.bindEvents()
|
am.bindEvents()
|
||||||
return am
|
return am
|
||||||
@@ -118,13 +108,9 @@ func NewAlertManager(app hubLike) *AlertManager {
|
|||||||
|
|
||||||
// Bind events to the alerts collection lifecycle
|
// Bind events to the alerts collection lifecycle
|
||||||
func (am *AlertManager) bindEvents() {
|
func (am *AlertManager) bindEvents() {
|
||||||
am.bindNetworkMonitorAlertEvents()
|
|
||||||
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
||||||
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
||||||
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
||||||
am.hub.OnRecordAfterCreateSuccess("zfs_pools").BindFunc(am.handleZfsPoolCreateAlert)
|
|
||||||
am.hub.OnRecordAfterUpdateSuccess("zfs_pools").BindFunc(am.handleZfsPoolAlert)
|
|
||||||
am.hub.OnRecordAfterDeleteSuccess("zfs_pools").BindFunc(resolveZfsPoolHistoryOnDelete)
|
|
||||||
|
|
||||||
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
|
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
|
||||||
// Populate all alerts into cache on startup
|
// Populate all alerts into cache on startup
|
||||||
@@ -133,9 +119,6 @@ func (am *AlertManager) bindEvents() {
|
|||||||
if err := resolveStatusAlerts(e.App); err != nil {
|
if err := resolveStatusAlerts(e.App); err != nil {
|
||||||
e.App.Logger().Error("Failed to resolve stale status alerts", "err", err)
|
e.App.Logger().Error("Failed to resolve stale status alerts", "err", err)
|
||||||
}
|
}
|
||||||
if err := resolveSystemdAlerts(e.App); err != nil {
|
|
||||||
e.App.Logger().Error("Failed to resolve stale systemd alerts", "err", err)
|
|
||||||
}
|
|
||||||
if err := am.restorePendingStatusAlerts(); err != nil {
|
if err := am.restorePendingStatusAlerts(); err != nil {
|
||||||
e.App.Logger().Error("Failed to restore pending status alerts", "err", err)
|
e.App.Logger().Error("Failed to restore pending status alerts", "err", err)
|
||||||
}
|
}
|
||||||
@@ -235,20 +218,8 @@ func (am *AlertManager) SendAlert(data AlertMessageData) error {
|
|||||||
am.hub.Logger().Error("Failed to unmarshal user settings", "err", err)
|
am.hub.Logger().Error("Failed to unmarshal user settings", "err", err)
|
||||||
}
|
}
|
||||||
// send alerts via webhooks
|
// send alerts via webhooks
|
||||||
send := sendPublicNotification
|
|
||||||
if len(userAlertSettings.Webhooks) > 0 {
|
|
||||||
// Read the owner's current role at delivery time, including for URLs
|
|
||||||
// saved before an admin was demoted. Never fall back on lookup failure.
|
|
||||||
owner, err := am.hub.FindRecordById("users", data.UserID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("load notification owner: %w", err)
|
|
||||||
}
|
|
||||||
if owner.GetString("role") == "admin" {
|
|
||||||
send = shoutrrr.Send
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, webhook := range userAlertSettings.Webhooks {
|
for _, webhook := range userAlertSettings.Webhooks {
|
||||||
if err := am.sendShoutrrrAlert(webhook, data.Title, data.Message, data.Link, data.LinkText, send); err != nil {
|
if err := am.SendShoutrrrAlert(webhook, data.Title, data.Message, data.Link, data.LinkText); err != nil {
|
||||||
am.hub.Logger().Error("Failed to send shoutrrr alert", "err", err)
|
am.hub.Logger().Error("Failed to send shoutrrr alert", "err", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -279,10 +250,6 @@ func (am *AlertManager) SendAlert(data AlertMessageData) error {
|
|||||||
|
|
||||||
// SendShoutrrrAlert sends an alert via a Shoutrrr URL
|
// SendShoutrrrAlert sends an alert via a Shoutrrr URL
|
||||||
func (am *AlertManager) SendShoutrrrAlert(notificationUrl, title, message, link, linkText string) error {
|
func (am *AlertManager) SendShoutrrrAlert(notificationUrl, title, message, link, linkText string) error {
|
||||||
return am.sendShoutrrrAlert(notificationUrl, title, message, link, linkText, shoutrrr.Send)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) sendShoutrrrAlert(notificationUrl, title, message, link, linkText string, send func(string, string) error) error {
|
|
||||||
// Parse the URL
|
// Parse the URL
|
||||||
parsedURL, err := url.Parse(notificationUrl)
|
parsedURL, err := url.Parse(notificationUrl)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -325,7 +292,7 @@ func (am *AlertManager) sendShoutrrrAlert(notificationUrl, title, message, link,
|
|||||||
parsedURL.RawQuery = queryParams.Encode()
|
parsedURL.RawQuery = queryParams.Encode()
|
||||||
// log.Println("URL after modification:", parsedURL.String())
|
// log.Println("URL after modification:", parsedURL.String())
|
||||||
|
|
||||||
err = send(parsedURL.String(), message)
|
err = shoutrrr.Send(parsedURL.String(), message)
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
am.hub.Logger().Info("Sent shoutrrr alert", "title", title)
|
am.hub.Logger().Info("Sent shoutrrr alert", "title", title)
|
||||||
|
|||||||
@@ -3,11 +3,12 @@ package alerts
|
|||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/hub/utils"
|
|
||||||
"github.com/nicholas-fedor/shoutrrr"
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
)
|
)
|
||||||
@@ -29,13 +30,6 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
|
|||||||
return e.BadRequestError("Bad data", err)
|
return e.BadRequestError("Bad data", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if reqData.Name == alertNameNetworkMonitorLoss {
|
|
||||||
if reqData.Value < 0 || reqData.Value >= 100 {
|
|
||||||
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
|
|
||||||
}
|
|
||||||
reqData.Min = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
|
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -43,9 +37,6 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
|
|||||||
|
|
||||||
err = e.App.RunInTransaction(func(txApp core.App) error {
|
err = e.App.RunInTransaction(func(txApp core.App) error {
|
||||||
for _, systemId := range reqData.Systems {
|
for _, systemId := range reqData.Systems {
|
||||||
if !userHasSystem(txApp, userID, systemId) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// find existing matching alert
|
// find existing matching alert
|
||||||
alertRecord, err := txApp.FindFirstRecordByFilter(alertsCollection,
|
alertRecord, err := txApp.FindFirstRecordByFilter(alertsCollection,
|
||||||
"system={:system} && name={:name} && user={:user}",
|
"system={:system} && name={:name} && user={:user}",
|
||||||
@@ -103,9 +94,6 @@ func DeleteUserAlerts(e *core.RequestEvent) error {
|
|||||||
|
|
||||||
err = e.App.RunInTransaction(func(txApp core.App) error {
|
err = e.App.RunInTransaction(func(txApp core.App) error {
|
||||||
for _, systemId := range reqData.Systems {
|
for _, systemId := range reqData.Systems {
|
||||||
if !userHasSystem(txApp, userID, systemId) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Find existing alert to delete
|
// Find existing alert to delete
|
||||||
alertRecord, err := txApp.FindFirstRecordByFilter("alerts",
|
alertRecord, err := txApp.FindFirstRecordByFilter("alerts",
|
||||||
"system={:system} && name={:name} && user={:user}",
|
"system={:system} && name={:name} && user={:user}",
|
||||||
@@ -134,15 +122,6 @@ func DeleteUserAlerts(e *core.RequestEvent) error {
|
|||||||
return e.JSON(http.StatusOK, map[string]any{"success": true, "count": numDeleted})
|
return e.JSON(http.StatusOK, map[string]any{"success": true, "count": numDeleted})
|
||||||
}
|
}
|
||||||
|
|
||||||
func userHasSystem(app core.App, userID, systemID string) bool {
|
|
||||||
system, err := app.FindRecordById("systems", systemID)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
shareAll, _ := utils.GetEnv("SHARE_ALL_SYSTEMS")
|
|
||||||
return shareAll == "true" || slices.Contains(system.GetStringSlice("users"), userID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SendTestNotification handles API request to send a test notification to a specified Shoutrrr URL
|
// SendTestNotification handles API request to send a test notification to a specified Shoutrrr URL
|
||||||
func (am *AlertManager) SendTestNotification(e *core.RequestEvent) error {
|
func (am *AlertManager) SendTestNotification(e *core.RequestEvent) error {
|
||||||
var data struct {
|
var data struct {
|
||||||
@@ -152,16 +131,62 @@ func (am *AlertManager) SendTestNotification(e *core.RequestEvent) error {
|
|||||||
if err != nil || data.URL == "" {
|
if err != nil || data.URL == "" {
|
||||||
return e.BadRequestError("URL is required", err)
|
return e.BadRequestError("URL is required", err)
|
||||||
}
|
}
|
||||||
send := shoutrrr.Send
|
// Only allow admins to send test notifications to internal URLs
|
||||||
if !e.Auth.IsSuperuser() && e.Auth.GetString("role") != "admin" {
|
if !e.Auth.IsSuperuser() && e.Auth.GetString("role") != "admin" {
|
||||||
send = sendPublicNotification
|
internalURL, err := isInternalURL(data.URL)
|
||||||
}
|
if err != nil {
|
||||||
err = am.sendShoutrrrAlert(data.URL, "Test Alert", "This is a notification from Beszel.", am.hub.Settings().Meta.AppURL, "View Beszel", send)
|
return e.BadRequestError(err.Error(), nil)
|
||||||
if errors.Is(err, errInternalDestination) || errors.Is(err, errUnrestrictedService) {
|
}
|
||||||
return e.ForbiddenError(err.Error(), nil)
|
if internalURL {
|
||||||
|
return e.ForbiddenError("Only admins can send to internal destinations", nil)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
err = am.SendShoutrrrAlert(data.URL, "Test Alert", "This is a notification from Beszel.", am.hub.Settings().Meta.AppURL, "View Beszel")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return e.JSON(200, map[string]string{"err": err.Error()})
|
return e.JSON(200, map[string]string{"err": err.Error()})
|
||||||
}
|
}
|
||||||
return e.JSON(200, map[string]bool{"err": false})
|
return e.JSON(200, map[string]bool{"err": false})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isInternalURL checks if the given shoutrrr URL points to an internal destination (localhost or private IP)
|
||||||
|
func isInternalURL(rawURL string) (bool, error) {
|
||||||
|
parsedURL, err := url.Parse(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
host := parsedURL.Hostname()
|
||||||
|
if host == "" {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.EqualFold(host, "localhost") {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if ip := net.ParseIP(host); ip != nil {
|
||||||
|
return isInternalIP(ip), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Some Shoutrrr URLs use the host position for service identifiers rather than a
|
||||||
|
// network hostname (for example, discord://token@webhookid). Restrict DNS lookups
|
||||||
|
// to names that look like actual hostnames so valid service URLs keep working.
|
||||||
|
if !strings.Contains(host, ".") {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ips, err := net.LookupIP(host)
|
||||||
|
if err != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if slices.ContainsFunc(ips, isInternalIP) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isInternalIP(ip net.IP) bool {
|
||||||
|
return ip.IsPrivate() || ip.IsLoopback() || ip.IsUnspecified()
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,11 +7,10 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/alerts"
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||||
pbTests "github.com/pocketbase/pocketbase/tests"
|
pbTests "github.com/pocketbase/pocketbase/tests"
|
||||||
|
|
||||||
@@ -30,6 +29,31 @@ func jsonReader(v any) io.Reader {
|
|||||||
return bytes.NewReader(data)
|
return bytes.NewReader(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsInternalURL(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
internal bool
|
||||||
|
}{
|
||||||
|
{name: "loopback ipv4", url: "generic://127.0.0.1", internal: true},
|
||||||
|
{name: "localhost hostname", url: "generic://localhost", internal: true},
|
||||||
|
{name: "localhost hostname", url: "generic+http://localhost/api/v1/postStuff", internal: true},
|
||||||
|
{name: "localhost hostname", url: "generic+http://127.0.0.1:8080/api/v1/postStuff", internal: true},
|
||||||
|
{name: "localhost hostname", url: "generic+https://beszel.dev/api/v1/postStuff", internal: false},
|
||||||
|
{name: "public ipv4", url: "generic://8.8.8.8", internal: false},
|
||||||
|
{name: "token style service url", url: "discord://abc123@123456789", internal: false},
|
||||||
|
{name: "single label service url", url: "slack://token@team/channel", internal: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
internal, err := alerts.IsInternalURL(testCase.url)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, testCase.internal, internal)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestUserAlertsApi(t *testing.T) {
|
func TestUserAlertsApi(t *testing.T) {
|
||||||
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
||||||
defer hub.Cleanup()
|
defer hub.Cleanup()
|
||||||
@@ -166,30 +190,6 @@ func TestUserAlertsApi(t *testing.T) {
|
|||||||
assert.EqualValues(t, 3, user1Alerts, "should have 3 alerts")
|
assert.EqualValues(t, 3, user1Alerts, "should have 3 alerts")
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
Name: "POST ignores systems the user cannot access",
|
|
||||||
Method: http.MethodPost,
|
|
||||||
URL: "/api/beszel/user-alerts",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Authorization": user2Token,
|
|
||||||
},
|
|
||||||
ExpectedStatus: 200,
|
|
||||||
ExpectedContent: []string{"\"success\":true"},
|
|
||||||
TestAppFactory: testAppFactory,
|
|
||||||
Body: jsonReader(map[string]any{
|
|
||||||
"name": "CPU",
|
|
||||||
"systems": []string{system1.Id},
|
|
||||||
"value": 90,
|
|
||||||
"min": 10,
|
|
||||||
}),
|
|
||||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
|
||||||
beszelTests.ClearCollection(t, app, "alerts")
|
|
||||||
},
|
|
||||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
|
||||||
alerts, _ := app.CountRecords("alerts")
|
|
||||||
assert.Zero(t, alerts)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
Name: "Overwrite: false, should not overwrite existing alert",
|
Name: "Overwrite: false, should not overwrite existing alert",
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
@@ -347,31 +347,6 @@ func TestUserAlertsApi(t *testing.T) {
|
|||||||
assert.Zero(t, alerts, "should have 0 alerts")
|
assert.Zero(t, alerts, "should have 0 alerts")
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
Name: "DELETE ignores systems the user cannot access",
|
|
||||||
Method: http.MethodDelete,
|
|
||||||
URL: "/api/beszel/user-alerts",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Authorization": user2Token,
|
|
||||||
},
|
|
||||||
ExpectedStatus: 200,
|
|
||||||
ExpectedContent: []string{"\"count\":0", "\"success\":true"},
|
|
||||||
TestAppFactory: testAppFactory,
|
|
||||||
Body: jsonReader(map[string]any{
|
|
||||||
"name": "CPU",
|
|
||||||
"systems": []string{system1.Id},
|
|
||||||
}),
|
|
||||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
|
||||||
beszelTests.ClearCollection(t, app, "alerts")
|
|
||||||
beszelTests.CreateRecord(app, "alerts", map[string]any{
|
|
||||||
"name": "CPU", "system": system1.Id, "user": user2.Id, "value": 80,
|
|
||||||
})
|
|
||||||
},
|
|
||||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
|
||||||
alerts, _ := app.CountRecords("alerts")
|
|
||||||
assert.EqualValues(t, 1, alerts)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
Name: "User 2 should not be able to delete alert of user 1",
|
Name: "User 2 should not be able to delete alert of user 1",
|
||||||
Method: http.MethodDelete,
|
Method: http.MethodDelete,
|
||||||
@@ -421,17 +396,6 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
hub, user := beszelTests.GetHubWithUser(t)
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
defer hub.Cleanup()
|
defer hub.Cleanup()
|
||||||
|
|
||||||
var delivered atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
delivered.Add(1)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
localURL := "generic+" + server.URL
|
|
||||||
|
|
||||||
readonlyUser, err := beszelTests.CreateUserWithRole(hub, "readonly@example.com", "password123", "readonly")
|
|
||||||
assert.NoError(t, err)
|
|
||||||
readonlyToken, err := readonlyUser.NewAuthToken()
|
|
||||||
assert.NoError(t, err)
|
|
||||||
userToken, err := user.NewAuthToken()
|
userToken, err := user.NewAuthToken()
|
||||||
|
|
||||||
adminUser, err := beszelTests.CreateUserWithRole(hub, "admin@example.com", "password123", "admin")
|
adminUser, err := beszelTests.CreateUserWithRole(hub, "admin@example.com", "password123", "admin")
|
||||||
@@ -456,11 +420,11 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
ExpectedContent: []string{"requires valid"},
|
ExpectedContent: []string{"requires valid"},
|
||||||
TestAppFactory: testAppFactory,
|
TestAppFactory: testAppFactory,
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": localURL,
|
"url": "generic://127.0.0.1",
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "POST /test-notification - invalid service reports error",
|
Name: "POST /test-notification - with external auth should succeed",
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
URL: "/api/beszel/test-notification",
|
URL: "/api/beszel/test-notification",
|
||||||
TestAppFactory: testAppFactory,
|
TestAppFactory: testAppFactory,
|
||||||
@@ -468,7 +432,7 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
"Authorization": userToken,
|
"Authorization": userToken,
|
||||||
},
|
},
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": "unknown://example.com",
|
"url": "generic://8.8.8.8",
|
||||||
}),
|
}),
|
||||||
ExpectedStatus: 200,
|
ExpectedStatus: 200,
|
||||||
ExpectedContent: []string{"\"err\":"},
|
ExpectedContent: []string{"\"err\":"},
|
||||||
@@ -510,10 +474,10 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
"Authorization": adminUserToken,
|
"Authorization": adminUserToken,
|
||||||
},
|
},
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": localURL,
|
"url": "generic://127.0.0.1",
|
||||||
}),
|
}),
|
||||||
ExpectedStatus: 200,
|
ExpectedStatus: 200,
|
||||||
ExpectedContent: []string{"\"err\":false"},
|
ExpectedContent: []string{"\"err\":"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "POST /test-notification - internal url with superuser auth should succeed",
|
Name: "POST /test-notification - internal url with superuser auth should succeed",
|
||||||
@@ -524,28 +488,14 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
"Authorization": superuserToken,
|
"Authorization": superuserToken,
|
||||||
},
|
},
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": localURL,
|
"url": "generic://127.0.0.1",
|
||||||
}),
|
}),
|
||||||
ExpectedStatus: 200,
|
ExpectedStatus: 200,
|
||||||
ExpectedContent: []string{"\"err\":"},
|
ExpectedContent: []string{"\"err\":"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} {
|
|
||||||
scenarios = append(scenarios, beszelTests.ApiScenario{
|
|
||||||
Name: "readonly cannot send to " + url,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
URL: "/api/beszel/test-notification",
|
|
||||||
TestAppFactory: testAppFactory,
|
|
||||||
Headers: map[string]string{"Authorization": readonlyToken},
|
|
||||||
Body: jsonReader(map[string]any{"url": url}),
|
|
||||||
ExpectedStatus: 403,
|
|
||||||
ExpectedContent: []string{"Only admins"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, scenario := range scenarios {
|
for _, scenario := range scenarios {
|
||||||
scenario.Test(t)
|
scenario.Test(t)
|
||||||
}
|
}
|
||||||
assert.EqualValues(t, 2, delivered.Load(), "only admin and superuser requests should reach the server")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
package alerts
|
package alerts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
"github.com/pocketbase/pocketbase/tools/store"
|
"github.com/pocketbase/pocketbase/tools/store"
|
||||||
@@ -11,17 +8,13 @@ import (
|
|||||||
|
|
||||||
// CachedAlertData represents the relevant fields of an alert record for status checking and updates.
|
// CachedAlertData represents the relevant fields of an alert record for status checking and updates.
|
||||||
type CachedAlertData struct {
|
type CachedAlertData struct {
|
||||||
Id string
|
Id string
|
||||||
SystemID string
|
SystemID string
|
||||||
UserID string
|
UserID string
|
||||||
Name string
|
Name string
|
||||||
Value float64
|
Value float64
|
||||||
Triggered bool
|
Triggered bool
|
||||||
Min uint8
|
Min uint8
|
||||||
PendingSince time.Time
|
|
||||||
// Immutable after publication; decoded only when the alert record changes.
|
|
||||||
MonitorStates map[string]string
|
|
||||||
MonitorStatesValid bool
|
|
||||||
// Created types.DateTime
|
// Created types.DateTime
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,19 +26,11 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
|
|||||||
a.Value = record.GetFloat("value")
|
a.Value = record.GetFloat("value")
|
||||||
a.Triggered = record.GetBool("triggered")
|
a.Triggered = record.GetBool("triggered")
|
||||||
a.Min = uint8(record.GetInt("min"))
|
a.Min = uint8(record.GetInt("min"))
|
||||||
a.PendingSince = record.GetDateTime("pending_since").Time()
|
|
||||||
if a.Name == alertNameNetworkMonitorLoss {
|
|
||||||
var state networkMonitorAlertState
|
|
||||||
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
|
|
||||||
a.MonitorStates = state.Monitors
|
|
||||||
}
|
|
||||||
// a.Created = record.GetDateTime("created")
|
// a.Created = record.GetDateTime("created")
|
||||||
}
|
}
|
||||||
|
|
||||||
// AlertsCache provides an in-memory cache for system alerts.
|
// AlertsCache provides an in-memory cache for system alerts.
|
||||||
type AlertsCache struct {
|
type AlertsCache struct {
|
||||||
// Serialize lazy loads with updates so a late load cannot replace newer state.
|
|
||||||
loadMu sync.Mutex
|
|
||||||
app core.App
|
app core.App
|
||||||
store *store.Store[string, *store.Store[string, CachedAlertData]]
|
store *store.Store[string, *store.Store[string, CachedAlertData]]
|
||||||
populated bool
|
populated bool
|
||||||
@@ -80,8 +65,6 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
|
|||||||
|
|
||||||
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
|
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
|
||||||
func (c *AlertsCache) PopulateFromDB(force bool) error {
|
func (c *AlertsCache) PopulateFromDB(force bool) error {
|
||||||
c.loadMu.Lock()
|
|
||||||
defer c.loadMu.Unlock()
|
|
||||||
if !force && c.populated {
|
if !force && c.populated {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -91,7 +74,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
|
|||||||
}
|
}
|
||||||
c.store.RemoveAll()
|
c.store.RemoveAll()
|
||||||
for _, record := range records {
|
for _, record := range records {
|
||||||
c.update(record)
|
c.Update(record)
|
||||||
}
|
}
|
||||||
c.populated = true
|
c.populated = true
|
||||||
return nil
|
return nil
|
||||||
@@ -99,12 +82,6 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
|
|||||||
|
|
||||||
// Update adds or updates an alert record in the cache.
|
// Update adds or updates an alert record in the cache.
|
||||||
func (c *AlertsCache) Update(record *core.Record) {
|
func (c *AlertsCache) Update(record *core.Record) {
|
||||||
c.loadMu.Lock()
|
|
||||||
defer c.loadMu.Unlock()
|
|
||||||
c.update(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *AlertsCache) update(record *core.Record) {
|
|
||||||
systemID := record.GetString("system")
|
systemID := record.GetString("system")
|
||||||
if systemID == "" {
|
if systemID == "" {
|
||||||
return
|
return
|
||||||
@@ -121,8 +98,6 @@ func (c *AlertsCache) update(record *core.Record) {
|
|||||||
|
|
||||||
// Delete removes an alert record from the cache.
|
// Delete removes an alert record from the cache.
|
||||||
func (c *AlertsCache) Delete(record *core.Record) {
|
func (c *AlertsCache) Delete(record *core.Record) {
|
||||||
c.loadMu.Lock()
|
|
||||||
defer c.loadMu.Unlock()
|
|
||||||
systemID := record.GetString("system")
|
systemID := record.GetString("system")
|
||||||
if systemID == "" {
|
if systemID == "" {
|
||||||
return
|
return
|
||||||
@@ -136,23 +111,18 @@ func (c *AlertsCache) Delete(record *core.Record) {
|
|||||||
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
|
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
|
||||||
systemStore, ok := c.store.GetOk(systemID)
|
systemStore, ok := c.store.GetOk(systemID)
|
||||||
if !ok {
|
if !ok {
|
||||||
c.loadMu.Lock()
|
// Populate cache for this system
|
||||||
defer c.loadMu.Unlock()
|
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
|
||||||
systemStore, ok = c.store.GetOk(systemID)
|
if err != nil {
|
||||||
if !ok {
|
return nil
|
||||||
// Populate cache for this system
|
|
||||||
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
systemStore = store.New(map[string]CachedAlertData{})
|
|
||||||
for _, record := range records {
|
|
||||||
var ca CachedAlertData
|
|
||||||
ca.PopulateFromRecord(record)
|
|
||||||
systemStore.Set(record.Id, ca)
|
|
||||||
}
|
|
||||||
c.store.Set(systemID, systemStore)
|
|
||||||
}
|
}
|
||||||
|
systemStore = store.New(map[string]CachedAlertData{})
|
||||||
|
for _, record := range records {
|
||||||
|
var ca CachedAlertData
|
||||||
|
ca.PopulateFromRecord(record)
|
||||||
|
systemStore.Set(record.Id, ca)
|
||||||
|
}
|
||||||
|
c.store.Set(systemID, systemStore)
|
||||||
}
|
}
|
||||||
all := systemStore.GetAll()
|
all := systemStore.GetAll()
|
||||||
alerts := make([]CachedAlertData, 0, len(all))
|
alerts := make([]CachedAlertData, 0, len(all))
|
||||||
|
|||||||
@@ -1,318 +0,0 @@
|
|||||||
package alerts
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// containerAlertName is the value stored in the alerts.name field for this alert type.
|
|
||||||
containerAlertName = "ContainerHealth"
|
|
||||||
|
|
||||||
// containerLogMaxLines caps how many matched (error/fatal) log lines are kept.
|
|
||||||
containerLogMaxLines = 12
|
|
||||||
// containerLogFallbackLines is how many trailing raw log lines are used when no
|
|
||||||
// line matches "error" or "fatal", so the notification still carries some context.
|
|
||||||
containerLogFallbackLines = 6
|
|
||||||
// containerLogExcerptMaxChars bounds a single container's log excerpt so a
|
|
||||||
// handful of containers can't blow past Discord's message size limit.
|
|
||||||
containerLogExcerptMaxChars = 500
|
|
||||||
// containerAlertMaxLogged is the max number of unhealthy containers we fetch
|
|
||||||
// and embed logs for in a single alert message.
|
|
||||||
containerAlertMaxLogged = 2
|
|
||||||
// containerAlertMessageMaxChars is a final safety cap on the whole message body.
|
|
||||||
containerAlertMessageMaxChars = 1800
|
|
||||||
)
|
|
||||||
|
|
||||||
// FetchContainerLogsFunc retrieves recent logs for a container ID from its
|
|
||||||
// connected agent. Implementations should apply their own timeout. This is a
|
|
||||||
// type alias (not a defined type) so it satisfies the hubLike interface in
|
|
||||||
// internal/hub/systems, which declares the same func signature without
|
|
||||||
// importing this package.
|
|
||||||
type FetchContainerLogsFunc = func(containerID string) (string, error)
|
|
||||||
|
|
||||||
// containerAlertTarget is an immutable snapshot of the fields needed after the
|
|
||||||
// alert fires. Keeping agent-owned container records out of notification work
|
|
||||||
// avoids retaining and concurrently reading data that is refreshed in place.
|
|
||||||
type containerAlertTarget struct {
|
|
||||||
id string
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleContainerAlerts checks configured "ContainerHealth" alerts for a system
|
|
||||||
// against the Docker container health data included in the latest agent update.
|
|
||||||
// It persists when containers first become unhealthy, fires from a fresh poll
|
|
||||||
// once the configured delay has elapsed, and resolves once containers recover.
|
|
||||||
// fetchLogs is used when an alert actually fires so the notification can include
|
|
||||||
// a log excerpt (prioritizing lines containing "error"/"fatal") for context.
|
|
||||||
func (am *AlertManager) HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs FetchContainerLogsFunc) error {
|
|
||||||
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, containerAlertName)
|
|
||||||
if len(alerts) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if data.Containers == nil {
|
|
||||||
// An unknown Docker state must not resolve a triggered alert or count
|
|
||||||
// toward the minimum unhealthy duration.
|
|
||||||
var result error
|
|
||||||
for _, alertData := range alerts {
|
|
||||||
if err := am.clearPendingContainerAlert(alertData); err != nil {
|
|
||||||
result = errors.Join(result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
var unhealthy []*container.Stats
|
|
||||||
for _, c := range data.Containers {
|
|
||||||
if c.Health == container.DockerHealthUnhealthy {
|
|
||||||
unhealthy = append(unhealthy, c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
systemName := systemRecord.GetString("name")
|
|
||||||
now := time.Now().UTC()
|
|
||||||
var result error
|
|
||||||
for _, alertData := range alerts {
|
|
||||||
if len(unhealthy) > 0 {
|
|
||||||
if alertData.Triggered {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
min := max(1, int(alertData.Min))
|
|
||||||
if alertData.PendingSince.IsZero() {
|
|
||||||
pendingSince, err := am.setPendingContainerAlert(alertData, now)
|
|
||||||
if err != nil {
|
|
||||||
result = errors.Join(result, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if pendingSince.IsZero() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
alertData.PendingSince = pendingSince
|
|
||||||
if min > 1 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if min > 1 && now.Before(alertData.PendingSince.Add(time.Duration(min)*time.Minute)) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := am.sendContainerHealthAlert(true, systemName, alertData, snapshotContainerAlertTargets(unhealthy), fetchLogs); err != nil {
|
|
||||||
result = errors.Join(result, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// no unhealthy containers right now
|
|
||||||
if err := am.clearPendingContainerAlert(alertData); err != nil {
|
|
||||||
result = errors.Join(result, err)
|
|
||||||
}
|
|
||||||
if !alertData.Triggered {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := am.sendContainerHealthAlert(false, systemName, alertData, nil, fetchLogs); err != nil {
|
|
||||||
result = errors.Join(result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotContainerAlertTargets(containers []*container.Stats) []containerAlertTarget {
|
|
||||||
targets := make([]containerAlertTarget, len(containers))
|
|
||||||
for i, c := range containers {
|
|
||||||
targets[i] = containerAlertTarget{id: c.Id, name: c.Name}
|
|
||||||
}
|
|
||||||
return targets
|
|
||||||
}
|
|
||||||
|
|
||||||
// setPendingContainerAlert durably records the first unhealthy observation and
|
|
||||||
// returns the persisted generation used to claim delivery.
|
|
||||||
func (am *AlertManager) setPendingContainerAlert(alertData CachedAlertData, since time.Time) (time.Time, error) {
|
|
||||||
record, err := am.hub.FindRecordById("alerts", alertData.Id)
|
|
||||||
if err != nil {
|
|
||||||
return time.Time{}, err
|
|
||||||
}
|
|
||||||
if record.GetBool("triggered") {
|
|
||||||
return time.Time{}, nil
|
|
||||||
}
|
|
||||||
if pendingSince := record.GetDateTime("pending_since").Time(); !pendingSince.IsZero() {
|
|
||||||
return pendingSince, nil
|
|
||||||
}
|
|
||||||
// PocketBase date fields are persisted with millisecond precision. Normalize
|
|
||||||
// before saving so the update-hook cache and a subsequent database read agree.
|
|
||||||
since = since.Truncate(time.Millisecond)
|
|
||||||
record.Set("pending_since", since)
|
|
||||||
return since, am.hub.Save(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) clearPendingContainerAlert(alertData CachedAlertData) error {
|
|
||||||
if alertData.PendingSince.IsZero() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
record, err := am.hub.FindRecordById("alerts", alertData.Id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if record.GetDateTime("pending_since").Time().IsZero() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
record.Set("pending_since", nil)
|
|
||||||
return am.hub.Save(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
// claimPendingContainerAlert marks an alert triggered only if the pending
|
|
||||||
// generation is still current. A healthy/unknown update can clear the timestamp
|
|
||||||
// while logs are being fetched, causing this claim to become a no-op.
|
|
||||||
func (am *AlertManager) claimPendingContainerAlert(alertData CachedAlertData) (bool, error) {
|
|
||||||
record, err := am.hub.FindRecordById("alerts", alertData.Id)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
pendingSince := record.GetDateTime("pending_since").Time()
|
|
||||||
if record.GetBool("triggered") || pendingSince.IsZero() || pendingSince.UnixMilli() != alertData.PendingSince.UnixMilli() {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
record.Set("pending_since", nil)
|
|
||||||
record.Set("triggered", true)
|
|
||||||
return true, am.hub.Save(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CancelPendingContainerAlerts clears pending container-health durations for a
|
|
||||||
// system. Called when monitoring pauses or the system goes down.
|
|
||||||
func (am *AlertManager) CancelPendingContainerAlerts(systemID string) {
|
|
||||||
for _, alertData := range am.alertsCache.GetAlertsByName(systemID, containerAlertName) {
|
|
||||||
if err := am.clearPendingContainerAlert(alertData); err != nil {
|
|
||||||
am.hub.Logger().Error("Failed to clear pending container alert", "err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendContainerHealthAlert updates the alert's triggered state and sends the
|
|
||||||
// notification. When unhealthy is true, it embeds a log excerpt (prioritizing
|
|
||||||
// error/fatal lines) for up to containerAlertMaxLogged of the affected containers.
|
|
||||||
func (am *AlertManager) sendContainerHealthAlert(unhealthy bool, systemName string, alertData CachedAlertData, containers []containerAlertTarget, fetchLogs FetchContainerLogsFunc) error {
|
|
||||||
link := am.hub.MakeLink("system", alertData.SystemID)
|
|
||||||
linkText := "View " + systemName
|
|
||||||
|
|
||||||
if !unhealthy {
|
|
||||||
if err := am.setAlertTriggered(alertData, false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
title := fmt.Sprintf("%s containers are healthy ✅", systemName)
|
|
||||||
return am.SendAlert(AlertMessageData{
|
|
||||||
UserID: alertData.UserID,
|
|
||||||
SystemID: alertData.SystemID,
|
|
||||||
Title: title,
|
|
||||||
Message: strings.TrimSuffix(title, " ✅"),
|
|
||||||
Link: link,
|
|
||||||
LinkText: linkText,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
names := make([]string, len(containers))
|
|
||||||
for i, c := range containers {
|
|
||||||
names[i] = c.name
|
|
||||||
}
|
|
||||||
|
|
||||||
var title string
|
|
||||||
if len(names) == 1 {
|
|
||||||
title = fmt.Sprintf("Unhealthy container %s on %s \U0001F534", names[0], systemName)
|
|
||||||
} else {
|
|
||||||
title = fmt.Sprintf("%d unhealthy containers on %s \U0001F534", len(names), systemName)
|
|
||||||
}
|
|
||||||
|
|
||||||
var body strings.Builder
|
|
||||||
fmt.Fprintf(&body, "Unhealthy: %s", strings.Join(names, ", "))
|
|
||||||
body.WriteString(am.buildContainerLogsSection(containers, fetchLogs))
|
|
||||||
|
|
||||||
message := body.String()
|
|
||||||
if len(message) > containerAlertMessageMaxChars {
|
|
||||||
message = message[:containerAlertMessageMaxChars] + "\n…(truncated)"
|
|
||||||
}
|
|
||||||
|
|
||||||
claimed, err := am.claimPendingContainerAlert(alertData)
|
|
||||||
if err != nil || !claimed {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return am.SendAlert(AlertMessageData{
|
|
||||||
UserID: alertData.UserID,
|
|
||||||
SystemID: alertData.SystemID,
|
|
||||||
Title: title,
|
|
||||||
Message: message,
|
|
||||||
Link: link,
|
|
||||||
LinkText: linkText,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildContainerLogsSection attempts to fetch and format log excerpts for up to
|
|
||||||
// containerAlertMaxLogged unhealthy containers, to append to an alert message.
|
|
||||||
func (am *AlertManager) buildContainerLogsSection(containers []containerAlertTarget, fetchLogs FetchContainerLogsFunc) string {
|
|
||||||
if fetchLogs == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
var section strings.Builder
|
|
||||||
attempts := min(len(containers), containerAlertMaxLogged)
|
|
||||||
for _, c := range containers[:attempts] {
|
|
||||||
rawLogs, err := fetchLogs(c.id)
|
|
||||||
if err != nil {
|
|
||||||
am.hub.Logger().Warn("Failed to fetch container logs for alert", "container", c.name, "err", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
excerpt := buildContainerLogExcerpt(rawLogs)
|
|
||||||
if excerpt == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fmt.Fprintf(§ion, "\n\n%s logs:\n```\n%s\n```", c.name, excerpt)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(containers) > containerAlertMaxLogged {
|
|
||||||
fmt.Fprintf(§ion, "\n\n(+%d more unhealthy container(s), logs omitted)", len(containers)-containerAlertMaxLogged)
|
|
||||||
}
|
|
||||||
|
|
||||||
return section.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildContainerLogExcerpt filters raw container log output down to the lines
|
|
||||||
// most likely to explain why the container is unhealthy: lines containing
|
|
||||||
// "error" or "fatal" (case-insensitive) are preferred. If none match, the tail
|
|
||||||
// of the raw output is used instead so the notification still carries context.
|
|
||||||
func buildContainerLogExcerpt(raw string) string {
|
|
||||||
raw = strings.TrimSpace(raw)
|
|
||||||
if raw == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
lines := strings.Split(raw, "\n")
|
|
||||||
|
|
||||||
var matched []string
|
|
||||||
for _, line := range lines {
|
|
||||||
line = strings.TrimRight(line, "\r")
|
|
||||||
if line == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
lower := strings.ToLower(line)
|
|
||||||
if strings.Contains(lower, "error") || strings.Contains(lower, "fatal") {
|
|
||||||
matched = append(matched, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
selected := matched
|
|
||||||
if len(selected) == 0 {
|
|
||||||
start := max(0, len(lines)-containerLogFallbackLines)
|
|
||||||
selected = lines[start:]
|
|
||||||
} else if len(selected) > containerLogMaxLines {
|
|
||||||
selected = selected[len(selected)-containerLogMaxLines:]
|
|
||||||
}
|
|
||||||
|
|
||||||
excerpt := strings.TrimSpace(strings.Join(selected, "\n"))
|
|
||||||
if len(excerpt) > containerLogExcerptMaxChars {
|
|
||||||
excerpt = "…" + excerpt[len(excerpt)-containerLogExcerptMaxChars:]
|
|
||||||
}
|
|
||||||
return excerpt
|
|
||||||
}
|
|
||||||
@@ -1,349 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package alerts_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/alerts"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
type containerAlertTestFixture struct {
|
|
||||||
hub *beszelTests.TestHub
|
|
||||||
am *alerts.AlertManager
|
|
||||||
alertID string
|
|
||||||
systemRecord *core.Record
|
|
||||||
}
|
|
||||||
|
|
||||||
func newContainerAlertTestFixture(t *testing.T, min int) *containerAlertTestFixture {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
|
||||||
|
|
||||||
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
|
||||||
require.NoError(t, err)
|
|
||||||
systemRecord := systems[0]
|
|
||||||
|
|
||||||
userSettings, err := hub.FindFirstRecordByFilter("user_settings", "user={:user}", map[string]any{"user": user.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
userSettings.Set("settings", `{"emails":["test@example.com"],"webhooks":[]}`)
|
|
||||||
require.NoError(t, hub.Save(userSettings))
|
|
||||||
|
|
||||||
alertRecord, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
|
||||||
"name": "ContainerHealth",
|
|
||||||
"system": systemRecord.Id,
|
|
||||||
"user": user.Id,
|
|
||||||
"min": min,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "Alert should not be triggered initially")
|
|
||||||
|
|
||||||
return &containerAlertTestFixture{
|
|
||||||
hub: hub,
|
|
||||||
am: alerts.NewTestAlertManagerWithoutWorker(hub),
|
|
||||||
alertID: alertRecord.Id,
|
|
||||||
systemRecord: systemRecord,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *containerAlertTestFixture) cleanup() {
|
|
||||||
f.hub.Cleanup()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *containerAlertTestFixture) submit(t *testing.T, containers []*container.Stats, fetchLogs alerts.FetchContainerLogsFunc) {
|
|
||||||
t.Helper()
|
|
||||||
data := &system.CombinedData{Containers: containers}
|
|
||||||
require.NoError(t, f.am.HandleContainerAlerts(f.systemRecord, data, fetchLogs))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *containerAlertTestFixture) submitInvalid(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
require.NoError(t, f.am.HandleContainerAlerts(f.systemRecord, &system.CombinedData{}, nil))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *containerAlertTestFixture) assertTriggered(t *testing.T, triggered bool, message string) {
|
|
||||||
t.Helper()
|
|
||||||
alertRecord, err := f.hub.FindRecordById("alerts", f.alertID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, triggered, alertRecord.GetBool("triggered"), message)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *containerAlertTestFixture) assertPending(t *testing.T, pending bool) {
|
|
||||||
t.Helper()
|
|
||||||
alertRecord, err := f.hub.FindRecordById("alerts", f.alertID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, pending, !alertRecord.GetDateTime("pending_since").Time().IsZero())
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForContainerAlert(d time.Duration) {
|
|
||||||
time.Sleep(d)
|
|
||||||
synctest.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
func healthyContainer(name string) *container.Stats {
|
|
||||||
return &container.Stats{Name: name, Id: "abc123def456", Health: container.DockerHealthHealthy}
|
|
||||||
}
|
|
||||||
|
|
||||||
func unhealthyContainer(name string) *container.Stats {
|
|
||||||
return &container.Stats{Name: name, Id: "abc123def456", Health: container.DockerHealthUnhealthy}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertTriggersAndResolves(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 1)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("web")}, nil)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, true, "A one-minute alert should trigger on the first unhealthy update")
|
|
||||||
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend(), "An email should have been sent")
|
|
||||||
|
|
||||||
msg := fixture.hub.TestMailer.LastMessage()
|
|
||||||
assert.Contains(t, msg.Subject, "web", "Subject should name the unhealthy container")
|
|
||||||
assert.Contains(t, strings.ToLower(msg.Subject), "unhealthy")
|
|
||||||
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("web")}, nil)
|
|
||||||
fixture.assertPending(t, false)
|
|
||||||
|
|
||||||
fixture.submitInvalid(t)
|
|
||||||
fixture.assertTriggered(t, true, "An invalid container snapshot should not resolve the alert")
|
|
||||||
assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend(), "An invalid snapshot should not send a recovery")
|
|
||||||
|
|
||||||
fixture.submit(t, []*container.Stats{}, nil)
|
|
||||||
waitForContainerAlert(time.Second)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, false, "Alert should resolve once the container is healthy again")
|
|
||||||
assert.Equal(t, 2, fixture.hub.TestMailer.TotalSend(), "A second email should have been sent for the recovery")
|
|
||||||
assert.Contains(t, fixture.hub.TestMailer.LastMessage().Subject, " healthy")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertInvalidSnapshotCancelsPending(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 5)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
||||||
fixture.assertPending(t, true)
|
|
||||||
waitForContainerAlert(time.Minute)
|
|
||||||
fixture.submitInvalid(t)
|
|
||||||
fixture.assertPending(t, false)
|
|
||||||
waitForContainerAlert(10 * time.Minute)
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, false, "Stale unhealthy data should not trigger an alert")
|
|
||||||
fixture.assertPending(t, true)
|
|
||||||
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertSystemDownCancelsPending(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 5)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
// Use the hub's alert manager because the system-manager status hook invokes
|
|
||||||
// cancellation on that instance.
|
|
||||||
am := fixture.hub.GetAlertManager()
|
|
||||||
require.NoError(t, am.HandleContainerAlerts(
|
|
||||||
fixture.systemRecord,
|
|
||||||
&system.CombinedData{Containers: []*container.Stats{unhealthyContainer("db")}},
|
|
||||||
nil,
|
|
||||||
))
|
|
||||||
fixture.assertPending(t, true)
|
|
||||||
|
|
||||||
fixture.systemRecord.Set("status", "down")
|
|
||||||
require.NoError(t, fixture.hub.Save(fixture.systemRecord))
|
|
||||||
|
|
||||||
fixture.assertPending(t, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertResolvesBeforeMinDelayCancelsPending(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 5)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
||||||
waitForContainerAlert(time.Minute)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, false, "Alert should not fire until the min delay elapses")
|
|
||||||
fixture.assertPending(t, true)
|
|
||||||
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
|
|
||||||
|
|
||||||
// container recovers before the 5 minute delay elapses
|
|
||||||
fixture.submit(t, []*container.Stats{healthyContainer("db")}, nil)
|
|
||||||
waitForContainerAlert(10 * time.Minute)
|
|
||||||
fixture.submit(t, []*container.Stats{healthyContainer("db")}, nil)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, false, "Alert should remain untriggered")
|
|
||||||
fixture.assertPending(t, false)
|
|
||||||
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend(), "No email should be sent for a container that recovered before the delay")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertPreservesPendingDurationAcrossManagerRestart(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 2)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
||||||
waitForContainerAlert(30 * time.Second)
|
|
||||||
|
|
||||||
restarted := alerts.NewTestAlertManagerWithoutWorker(fixture.hub)
|
|
||||||
waitForContainerAlert(91 * time.Second)
|
|
||||||
require.NoError(t, restarted.HandleContainerAlerts(
|
|
||||||
fixture.systemRecord,
|
|
||||||
&system.CombinedData{Containers: []*container.Stats{unhealthyContainer("db")}},
|
|
||||||
nil,
|
|
||||||
))
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, true, "Restart should preserve the original unhealthy start time")
|
|
||||||
fixture.assertPending(t, false)
|
|
||||||
assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertClaimsPendingTimestampAtDatabasePrecision(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 1)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
alertRecord, err := fixture.hub.FindRecordById("alerts", fixture.alertID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
// PocketBase persists dates to milliseconds, while record update hooks can
|
|
||||||
// retain the original sub-millisecond value in the in-memory alert cache.
|
|
||||||
alertRecord.Set("pending_since", time.Now().UTC().Add(-2*time.Minute).Truncate(time.Millisecond).Add(123*time.Nanosecond))
|
|
||||||
require.NoError(t, fixture.hub.Save(alertRecord))
|
|
||||||
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, true, "Equivalent persisted and cached timestamps should claim the alert")
|
|
||||||
fixture.assertPending(t, false)
|
|
||||||
assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertRecoveryWhileFetchingLogsCancelsDelivery(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 1)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fetchLogs := func(containerID string) (string, error) {
|
|
||||||
fixture.submit(t, []*container.Stats{healthyContainer("api")}, nil)
|
|
||||||
return "FATAL stale failure", nil
|
|
||||||
}
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, false, "Recovery should cancel delivery while logs are fetched")
|
|
||||||
fixture.assertPending(t, false)
|
|
||||||
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertIncludesLogExcerpt(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 1)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
rawLogs := strings.Join([]string{
|
|
||||||
"2026-08-16T10:00:00Z booting",
|
|
||||||
"2026-08-16T10:00:01Z ERROR could not reach upstream",
|
|
||||||
"2026-08-16T10:00:02Z FATAL giving up after 3 retries",
|
|
||||||
}, "\n")
|
|
||||||
fetchLogs := func(containerID string) (string, error) {
|
|
||||||
assert.Equal(t, "abc123def456", containerID)
|
|
||||||
return rawLogs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, true, "Alert should be triggered")
|
|
||||||
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
||||||
|
|
||||||
body := fixture.hub.TestMailer.LastMessage().Text
|
|
||||||
assert.Contains(t, body, "could not reach upstream")
|
|
||||||
assert.Contains(t, body, "giving up after 3 retries")
|
|
||||||
assert.NotContains(t, body, "booting", "non error/fatal lines should be dropped when matches exist")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertSkipsLogsOnFetchError(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 1)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
fetchLogs := func(containerID string) (string, error) {
|
|
||||||
return "", fmt.Errorf("agent unreachable")
|
|
||||||
}
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, true, "Alert should still be triggered even if logs can't be fetched")
|
|
||||||
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContainerHealthAlertCapsLogFetchAttempts(t *testing.T) {
|
|
||||||
fixture := newContainerAlertTestFixture(t, 1)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
containers := make([]*container.Stats, 100)
|
|
||||||
for i := range containers {
|
|
||||||
containers[i] = &container.Stats{
|
|
||||||
Name: fmt.Sprintf("container-%d", i),
|
|
||||||
Id: fmt.Sprintf("id-%d", i),
|
|
||||||
Health: container.DockerHealthUnhealthy,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
attempts := 0
|
|
||||||
fetchLogs := func(containerID string) (string, error) {
|
|
||||||
attempts++
|
|
||||||
return "", fmt.Errorf("agent unreachable")
|
|
||||||
}
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
fixture.submit(t, containers, fetchLogs)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, true, "Alert should still fire when log retrieval fails")
|
|
||||||
assert.Equal(t, 2, attempts, "Log retrieval should attempt at most two containers")
|
|
||||||
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildContainerLogExcerptPrefersErrorAndFatalLines(t *testing.T) {
|
|
||||||
raw := strings.Join([]string{
|
|
||||||
"2026-08-16T10:00:00Z starting up",
|
|
||||||
"2026-08-16T10:00:01Z listening on :8080",
|
|
||||||
"2026-08-16T10:00:02Z ERROR failed to connect to db",
|
|
||||||
"2026-08-16T10:00:03Z retrying connection",
|
|
||||||
"2026-08-16T10:00:04Z FATAL could not recover, exiting",
|
|
||||||
}, "\n")
|
|
||||||
|
|
||||||
excerpt := alerts.BuildContainerLogExcerpt(raw)
|
|
||||||
assert.Contains(t, excerpt, "failed to connect to db")
|
|
||||||
assert.Contains(t, excerpt, "could not recover, exiting")
|
|
||||||
assert.NotContains(t, excerpt, "starting up", "non-matching lines should be dropped when error/fatal lines exist")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildContainerLogExcerptFallsBackToTailWhenNoMatches(t *testing.T) {
|
|
||||||
var lines []string
|
|
||||||
for i := range 20 {
|
|
||||||
lines = append(lines, fmt.Sprintf("line %d: all good here", i))
|
|
||||||
}
|
|
||||||
raw := strings.Join(lines, "\n")
|
|
||||||
|
|
||||||
excerpt := alerts.BuildContainerLogExcerpt(raw)
|
|
||||||
assert.Contains(t, excerpt, "line 19", "should keep the tail of the output")
|
|
||||||
assert.NotContains(t, excerpt, "line 0:", "should not keep the very start when falling back to a short tail")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildContainerLogExcerptEmpty(t *testing.T) {
|
|
||||||
assert.Equal(t, "", alerts.BuildContainerLogExcerpt(" \n \n"))
|
|
||||||
}
|
|
||||||
@@ -9,12 +9,6 @@ import (
|
|||||||
|
|
||||||
// On triggered alert record delete, set matching alert history record to resolved
|
// On triggered alert record delete, set matching alert history record to resolved
|
||||||
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
||||||
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
|
||||||
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
if !e.Record.GetBool("triggered") {
|
if !e.Record.GetBool("triggered") {
|
||||||
return e.Next()
|
return e.Next()
|
||||||
}
|
}
|
||||||
@@ -24,10 +18,6 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
|||||||
|
|
||||||
// On alert record update, update alert history record
|
// On alert record update, update alert history record
|
||||||
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
|
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
|
||||||
// Network monitor incidents have separate history entries per monitor.
|
|
||||||
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
original := e.Record.Original()
|
original := e.Record.Original()
|
||||||
new := e.Record
|
new := e.Record
|
||||||
|
|
||||||
|
|||||||
@@ -1,269 +0,0 @@
|
|||||||
package alerts
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"net"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
)
|
|
||||||
|
|
||||||
const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
|
|
||||||
|
|
||||||
// networkMonitorAlertState is this alert type's persisted runtime state.
|
|
||||||
// Monitor IDs map to their open history entries independently of history retention.
|
|
||||||
type networkMonitorAlertState struct {
|
|
||||||
Monitors map[string]string `json:"monitors"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
|
|
||||||
// Hidden fields are still writable through the record API unless protected.
|
|
||||||
protectState := func(e *core.RecordRequestEvent) error {
|
|
||||||
e.Record.Set("state", e.Record.Original().Get("state"))
|
|
||||||
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
|
|
||||||
if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
|
|
||||||
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
|
|
||||||
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
|
|
||||||
}
|
|
||||||
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
|
||||||
if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
|
|
||||||
return e.ForbiddenError("You do not have access to this system", nil)
|
|
||||||
}
|
|
||||||
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
|
|
||||||
value := e.Record.GetFloat("value")
|
|
||||||
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
|
|
||||||
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
|
|
||||||
}
|
|
||||||
e.Record.Set("min", 0)
|
|
||||||
}
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
|
|
||||||
am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
|
|
||||||
cleanup := func(e *core.RecordEvent) error {
|
|
||||||
if err := e.Next(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
|
|
||||||
}
|
|
||||||
am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
|
|
||||||
am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
|
||||||
if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
return cleanup(e)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
|
|
||||||
// using its exact payload (dashboard requests can replace the cached payload).
|
|
||||||
// Omitted results and disconnected systems never imply recovery.
|
|
||||||
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
|
|
||||||
if systemRecord.GetString("status") != "up" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
|
|
||||||
if len(alerts) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
monitors, err := am.networkMonitors.get(systemRecord.Id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// The cache only predicts a transition. Reload and recheck under the DB
|
|
||||||
// transaction before persisting, including current system/monitor status.
|
|
||||||
return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
|
|
||||||
}
|
|
||||||
|
|
||||||
// networkMonitorTransitionPending does no IO and never mutates cached maps.
|
|
||||||
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
|
|
||||||
for _, alert := range alerts {
|
|
||||||
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for id := range alert.MonitorStates {
|
|
||||||
if _, enabled := monitors[id]; !enabled {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for id, result := range results {
|
|
||||||
interval, enabled := monitors[id]
|
|
||||||
if !enabled || !monitorResultReady(result, interval, now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
_, active := alert.MonitorStates[id]
|
|
||||||
if (result.PacketLoss1h > alert.Value) != active {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
|
|
||||||
var messages []AlertMessageData
|
|
||||||
err := app.RunInTransaction(func(tx core.App) error {
|
|
||||||
// Read configuration inside the transaction so concurrent threshold changes,
|
|
||||||
// disabling, and evaluations cannot overwrite each other's incident state.
|
|
||||||
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
|
|
||||||
if err != nil || len(alerts) == 0 {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
system, err := tx.FindRecordById("systems", systemID)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
// System deletion cascades to its alerts.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
enabled := make(map[string]*core.Record, len(monitors))
|
|
||||||
for _, m := range monitors {
|
|
||||||
enabled[m.Id] = m
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
for _, alert := range alerts {
|
|
||||||
var state networkMonitorAlertState
|
|
||||||
if err := alert.UnmarshalJSONField("state", &state); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
states := state.Monitors
|
|
||||||
if states == nil {
|
|
||||||
states = map[string]string{}
|
|
||||||
}
|
|
||||||
changed := false
|
|
||||||
// Removing or disabling a monitor closes its incident silently.
|
|
||||||
for id, historyID := range states {
|
|
||||||
if _, ok := enabled[id]; !ok {
|
|
||||||
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
delete(states, id)
|
|
||||||
changed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if system.GetString("status") == "up" {
|
|
||||||
for _, m := range monitors {
|
|
||||||
result, ok := results[m.Id]
|
|
||||||
if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
historyID, active := states[m.Id]
|
|
||||||
triggered := result.PacketLoss1h > alert.GetFloat("value")
|
|
||||||
if triggered == active {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
label := m.GetString("target")
|
|
||||||
if m.GetString("protocol") == "tcp" {
|
|
||||||
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
|
|
||||||
}
|
|
||||||
if triggered {
|
|
||||||
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
history := core.NewRecord(collection)
|
|
||||||
history.Load(map[string]any{
|
|
||||||
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
|
|
||||||
"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
|
|
||||||
})
|
|
||||||
if err := tx.Save(history); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
states[m.Id] = history.Id
|
|
||||||
} else {
|
|
||||||
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
delete(states, m.Id)
|
|
||||||
}
|
|
||||||
changed = true
|
|
||||||
state, comparison := "loss", "exceeds"
|
|
||||||
if !triggered {
|
|
||||||
state, comparison = "recovered", "is at or below"
|
|
||||||
}
|
|
||||||
messages = append(messages, AlertMessageData{
|
|
||||||
UserID: alert.GetString("user"), SystemID: systemID,
|
|
||||||
Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
|
|
||||||
Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
|
|
||||||
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if changed || alert.GetBool("triggered") != (len(states) > 0) {
|
|
||||||
alert.Set("state", networkMonitorAlertState{Monitors: states})
|
|
||||||
alert.Set("triggered", len(states) > 0)
|
|
||||||
if err := tx.Save(alert); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Match other alert types: persist transitions before delivery, and respect
|
|
||||||
// the user's existing notification destinations and quiet hours.
|
|
||||||
for _, message := range messages {
|
|
||||||
if err := am.SendAlert(message); err != nil {
|
|
||||||
app.Logger().Error("Failed to send network monitor alert", "err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
|
|
||||||
// Three completed attempts provide a short warm-up, including after an agent
|
|
||||||
// restart.
|
|
||||||
if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
// Never interpret an empty one-hour window as zero loss.
|
|
||||||
maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
|
|
||||||
age := now.Sub(time.UnixMilli(result.LastProbeAt))
|
|
||||||
return age >= -time.Minute && age <= maxAge
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveMonitorIncident(app core.App, id string, now time.Time) error {
|
|
||||||
record, err := app.FindRecordById("alerts_history", id)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
// History can be purged independently.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !record.GetDateTime("resolved").IsZero() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
record.Set("resolved", now.UTC())
|
|
||||||
return app.Save(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveNetworkMonitorHistory(app core.App, alertID string) error {
|
|
||||||
records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, record := range records {
|
|
||||||
record.Set("resolved", time.Now().UTC())
|
|
||||||
if err := app.Save(record); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,513 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package alerts_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/alerts"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
pbTests "github.com/pocketbase/pocketbase/tests"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
|
|
||||||
t.Helper()
|
|
||||||
hub, system, alert := systemdTestSetup(t, false)
|
|
||||||
t.Cleanup(hub.Cleanup)
|
|
||||||
alert.Set("name", "NetworkMonitorLoss")
|
|
||||||
alert.Set("value", 5)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
var monitors []*core.Record
|
|
||||||
for _, name := range []string{"gateway", "website"} {
|
|
||||||
record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
|
|
||||||
"system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
monitors = append(monitors, record)
|
|
||||||
}
|
|
||||||
// Avoid starting a system update worker in tests.
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
system.Set("status", "up")
|
|
||||||
return hub, system, alert, monitors
|
|
||||||
}
|
|
||||||
|
|
||||||
func monitorResult(loss float64) monitor.Result {
|
|
||||||
return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
|
|
||||||
check := func(active bool, open, sent int) {
|
|
||||||
t.Helper()
|
|
||||||
record, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, active, record.GetBool("triggered"))
|
|
||||||
total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.EqualValues(t, open, total)
|
|
||||||
assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 1, 1)
|
|
||||||
message := hub.TestMailer.Messages()[count]
|
|
||||||
assert.Contains(t, message.Text, "gateway.example.com")
|
|
||||||
assert.Contains(t, message.Text, "10.00%")
|
|
||||||
assert.Contains(t, message.Text, "5.00%")
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 1, 1)
|
|
||||||
// Persisted monitor state prevents duplicate notifications after a hub restart.
|
|
||||||
am = alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results[monitors[1].Id] = monitorResult(20)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 2, 2)
|
|
||||||
results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 1, 3)
|
|
||||||
results[monitors[1].Id] = monitorResult(0)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(false, 0, 4)
|
|
||||||
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, histories, 2)
|
|
||||||
for _, history := range histories {
|
|
||||||
assert.NotEmpty(t, history.GetString("monitor_name"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
protocol, target, label string
|
|
||||||
port int
|
|
||||||
}{
|
|
||||||
{"icmp", "gateway.example.com", "gateway.example.com", 0},
|
|
||||||
{"http", "https://example.com/health", "https://example.com/health", 0},
|
|
||||||
{"tcp", "example.com", "example.com:8443", 8443},
|
|
||||||
{"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
|
|
||||||
} {
|
|
||||||
t.Run(tc.label, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
m := monitors[0]
|
|
||||||
m.Set("protocol", tc.protocol)
|
|
||||||
m.Set("target", tc.target)
|
|
||||||
m.Set("port", tc.port)
|
|
||||||
require.NoError(t, hub.Save(m))
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
|
|
||||||
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, histories, 1)
|
|
||||||
assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
|
|
||||||
assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
|
|
||||||
for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
|
|
||||||
t.Run(scenario, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
apply := func(loss float64) {
|
|
||||||
result := monitorResult(loss)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: result}
|
|
||||||
switch scenario {
|
|
||||||
case "missing":
|
|
||||||
results = nil
|
|
||||||
case "stale":
|
|
||||||
result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "expired hourly window":
|
|
||||||
monitors[0].Set("interval", 3600)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "future":
|
|
||||||
result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "warmup":
|
|
||||||
result.SampleCount = 2
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "no probes":
|
|
||||||
result.SampleCount = 0
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "down":
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
case "paused":
|
|
||||||
record, err := hub.FindRecordById("systems", system.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
record.Set("status", "paused")
|
|
||||||
require.NoError(t, hub.Save(record))
|
|
||||||
}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
}
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
apply(100)
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
|
|
||||||
apply(0)
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
|
|
||||||
record, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, record.GetBool("triggered"))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertCleanup(t *testing.T) {
|
|
||||||
for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
|
|
||||||
t.Run(scenario, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
switch scenario {
|
|
||||||
case "disable monitor":
|
|
||||||
monitors[0].Set("enabled", false)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
case "delete monitor":
|
|
||||||
require.NoError(t, hub.Delete(monitors[0]))
|
|
||||||
case "disable alert":
|
|
||||||
require.NoError(t, hub.Delete(alert))
|
|
||||||
case "delete system":
|
|
||||||
require.NoError(t, hub.Delete(system))
|
|
||||||
case "purge history":
|
|
||||||
history, err := hub.FindAllRecords("alerts_history")
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, record := range history {
|
|
||||||
require.NoError(t, hub.Delete(record))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
|
|
||||||
require.NoError(t, err)
|
|
||||||
if scenario == "disable monitor" || scenario == "delete monitor" {
|
|
||||||
assert.EqualValues(t, 1, open)
|
|
||||||
record, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, record.GetBool("triggered"))
|
|
||||||
require.NoError(t, hub.Delete(monitors[1]))
|
|
||||||
record, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, record.GetBool("triggered"))
|
|
||||||
} else {
|
|
||||||
assert.Zero(t, open)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password")
|
|
||||||
require.NoError(t, err)
|
|
||||||
other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
|
|
||||||
require.NoError(t, err)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
other, err = hub.FindRecordById("alerts", other.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, other.GetBool("triggered"))
|
|
||||||
// Editing the threshold re-evaluates on the next batch, without losing state.
|
|
||||||
alert, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
alert.Set("value", 15)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
alert, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alert.GetBool("triggered"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertAPI(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
value float64
|
|
||||||
direct, denied, patch bool
|
|
||||||
status int
|
|
||||||
}{
|
|
||||||
{name: "zero threshold", value: 0, status: 200},
|
|
||||||
{name: "fractional threshold", value: 5.5, status: 200},
|
|
||||||
{name: "negative threshold", value: -1, status: 400},
|
|
||||||
{name: "unreachable threshold", value: 100, status: 400},
|
|
||||||
{name: "bulk inaccessible system", value: 5, denied: true, status: 200},
|
|
||||||
{name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
|
|
||||||
{name: "direct invalid threshold", value: -1, direct: true, status: 400},
|
|
||||||
{name: "direct private state", value: 5, direct: true, status: 200},
|
|
||||||
{name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
owner := user.Id
|
|
||||||
if tc.denied {
|
|
||||||
other, err := beszelTests.CreateUser(hub, "other@example.com", "password")
|
|
||||||
require.NoError(t, err)
|
|
||||||
owner = other.Id
|
|
||||||
}
|
|
||||||
systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
|
|
||||||
require.NoError(t, err)
|
|
||||||
token, err := user.NewAuthToken()
|
|
||||||
require.NoError(t, err)
|
|
||||||
body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
|
|
||||||
url, method := "/api/beszel/user-alerts", "POST"
|
|
||||||
if tc.direct {
|
|
||||||
url = "/api/collections/alerts/records"
|
|
||||||
body["system"], body["user"] = systems[0].Id, user.Id
|
|
||||||
body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
|
|
||||||
}
|
|
||||||
if tc.patch {
|
|
||||||
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
|
|
||||||
require.NoError(t, err)
|
|
||||||
url += "/" + alert.Id
|
|
||||||
method = "PATCH"
|
|
||||||
body["triggered"] = false
|
|
||||||
}
|
|
||||||
content := `"success":true`
|
|
||||||
if tc.direct {
|
|
||||||
content = `"name":"NetworkMonitorLoss"`
|
|
||||||
}
|
|
||||||
if tc.status == 400 {
|
|
||||||
content = `"status":400`
|
|
||||||
}
|
|
||||||
if tc.status == 403 {
|
|
||||||
content = `"status":403`
|
|
||||||
}
|
|
||||||
scenario := beszelTests.ApiScenario{
|
|
||||||
Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
|
|
||||||
Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
|
|
||||||
TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
|
|
||||||
}
|
|
||||||
scenario.Test(t)
|
|
||||||
records, err := hub.FindAllRecords("alerts")
|
|
||||||
require.NoError(t, err)
|
|
||||||
if tc.status != 200 || tc.denied {
|
|
||||||
assert.Empty(t, records)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
require.Len(t, records, 1)
|
|
||||||
assert.Equal(t, tc.value, records[0].GetFloat("value"))
|
|
||||||
assert.Zero(t, records[0].GetInt("min"))
|
|
||||||
state := struct {
|
|
||||||
Monitors map[string]string `json:"monitors"`
|
|
||||||
}{}
|
|
||||||
require.NoError(t, records[0].UnmarshalJSONField("state", &state))
|
|
||||||
states := state.Monitors
|
|
||||||
if tc.patch {
|
|
||||||
assert.Equal(t, map[string]string{"real": "history"}, states)
|
|
||||||
assert.True(t, records[0].GetBool("triggered"))
|
|
||||||
} else {
|
|
||||||
assert.Empty(t, states)
|
|
||||||
assert.False(t, records[0].GetBool("triggered"))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type monitorCountingHub struct {
|
|
||||||
*beszelTests.TestHub
|
|
||||||
transactions atomic.Int64
|
|
||||||
beforeTransaction func()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
|
|
||||||
h.transactions.Add(1)
|
|
||||||
if h.beforeTransaction != nil {
|
|
||||||
h.beforeTransaction()
|
|
||||||
}
|
|
||||||
return h.App.RunInTransaction(fn)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Count actual SQL on both DB connections, including queries through record APIs.
|
|
||||||
func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
|
|
||||||
t.Helper()
|
|
||||||
count := &atomic.Int64{}
|
|
||||||
for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
|
|
||||||
db := builder.(*dbx.DB)
|
|
||||||
old := db.LogFunc
|
|
||||||
db.LogFunc = func(string, ...any) { count.Add(1) }
|
|
||||||
t.Cleanup(func() { db.LogFunc = old })
|
|
||||||
}
|
|
||||||
return count
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
counted := &monitorCountingHub{TestHub: hub}
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(counted)
|
|
||||||
sql := monitorSQLCounter(t, hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
|
|
||||||
noWork := func() {
|
|
||||||
t.Helper()
|
|
||||||
sql.Store(0)
|
|
||||||
counted.transactions.Store(0)
|
|
||||||
for range 100 {
|
|
||||||
evaluate()
|
|
||||||
}
|
|
||||||
assert.Zero(t, sql.Load(), "steady state must not issue SQL")
|
|
||||||
assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
|
|
||||||
}
|
|
||||||
// One-time lazy loads are permitted, including on hub restart.
|
|
||||||
evaluate()
|
|
||||||
assert.Positive(t, sql.Load())
|
|
||||||
noWork()
|
|
||||||
// Realtime metric saves invoke record hooks but must not invalidate config.
|
|
||||||
fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
monitors[0] = fresh
|
|
||||||
monitors[0].Set("loss1h", 0)
|
|
||||||
monitors[0].Set("res", 100)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
noWork()
|
|
||||||
results[monitors[0].Id] = monitorResult(10)
|
|
||||||
evaluate()
|
|
||||||
assert.Positive(t, sql.Load(), "transitions must still be persisted")
|
|
||||||
assert.EqualValues(t, 1, counted.transactions.Load())
|
|
||||||
noWork()
|
|
||||||
// Missing and stale observations must not enter the transaction either.
|
|
||||||
results = nil
|
|
||||||
noWork()
|
|
||||||
results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
|
|
||||||
noWork()
|
|
||||||
results[monitors[0].Id] = monitorResult(0)
|
|
||||||
evaluate()
|
|
||||||
noWork()
|
|
||||||
require.NoError(t, hub.Delete(alert))
|
|
||||||
noWork()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
// Widening the interval makes this observation fresh. A stale interval cache
|
|
||||||
// would miss the failure indefinitely, even though results keep arriving.
|
|
||||||
result := monitorResult(10)
|
|
||||||
result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
monitors[0].Set("interval", 120)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
|
||||||
// Disable, then re-enable the same ID: its new failure must be detected.
|
|
||||||
monitors[0].Set("enabled", false)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
monitors[0].Set("enabled", true)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+2, hub.TestMailer.TotalSend())
|
|
||||||
// A new monitor must also become eligible without restarting the hub.
|
|
||||||
created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
|
|
||||||
"system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
results[created.Id] = monitorResult(10)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+3, hub.TestMailer.TotalSend())
|
|
||||||
// Threshold changes refresh cached config and preserve the active incidents.
|
|
||||||
alert, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
alert.Set("value", 15)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+5, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
|
|
||||||
for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
|
|
||||||
t.Run(change, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
counted := &monitorCountingHub{TestHub: hub}
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(counted)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
// Change the DB after the cache predicts a transition, before its transaction.
|
|
||||||
counted.beforeTransaction = func() {
|
|
||||||
counted.beforeTransaction = nil
|
|
||||||
switch change {
|
|
||||||
case "threshold":
|
|
||||||
alert.Set("value", 20)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
case "disable alert":
|
|
||||||
require.NoError(t, hub.Delete(alert))
|
|
||||||
case "disable monitor":
|
|
||||||
monitors[0].Set("enabled", false)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
case "down":
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
results[monitors[0].Id] = monitorResult(10)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.EqualValues(t, 1, counted.transactions.Load())
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
histories, err := hub.CountRecords("alerts_history")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Zero(t, histories)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
|
|
||||||
hub, system, _, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
errs := make(chan error, 8)
|
|
||||||
for range 8 {
|
|
||||||
wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(errs)
|
|
||||||
for err := range errs {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
|
|
||||||
hub, system, _, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
_, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
results[monitors[0].Id] = monitorResult(10)
|
|
||||||
require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
histories, err := hub.CountRecords("alerts_history")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Zero(t, histories)
|
|
||||||
_, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
// A failed transition must not be published to the cache and mask the retry.
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
@@ -13,41 +13,8 @@ import (
|
|||||||
"github.com/pocketbase/pocketbase/tools/types"
|
"github.com/pocketbase/pocketbase/tools/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
var cpuStateAlerts = map[string]struct {
|
|
||||||
index int
|
|
||||||
label string
|
|
||||||
}{
|
|
||||||
"CPUIOWait": {2, "CPU I/O Wait"},
|
|
||||||
"CPUSteal": {3, "CPU Steal Time"},
|
|
||||||
}
|
|
||||||
|
|
||||||
func cpuStateAlertValue(name string, breakdown []float64) (float64, bool) {
|
|
||||||
state, ok := cpuStateAlerts[name]
|
|
||||||
if !ok || len(breakdown) < 5 {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
var total float64
|
|
||||||
for _, value := range breakdown {
|
|
||||||
total += value
|
|
||||||
}
|
|
||||||
if total <= 0 {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
return breakdown[state.index], true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error {
|
func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error {
|
||||||
// Systemd alerts are binary state, not numeric thresholds, so they're handled
|
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status")
|
||||||
// separately. They read their own state from the database and don't use data.
|
|
||||||
if err := am.HandleSystemdAlerts(systemRecord); err != nil {
|
|
||||||
am.hub.Logger().Error("Error handling systemd alerts", "err", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if data == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
|
|
||||||
if len(alerts) == 0 {
|
if len(alerts) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -77,14 +44,6 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
maxUsedPct = usedPct
|
maxUsedPct = usedPct
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, pool := range data.Stats.ZfsPools {
|
|
||||||
if pool != nil && !pool.Raw && pool.Total > 0 {
|
|
||||||
usedPct := pool.Used / pool.Total * 100
|
|
||||||
if usedPct > maxUsedPct {
|
|
||||||
maxUsedPct = usedPct
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
val = maxUsedPct
|
val = maxUsedPct
|
||||||
case "Temperature":
|
case "Temperature":
|
||||||
if data.Info.DashboardTemp < 1 {
|
if data.Info.DashboardTemp < 1 {
|
||||||
@@ -108,11 +67,6 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
val = float64(data.Stats.Battery[0])
|
val = float64(data.Stats.Battery[0])
|
||||||
default:
|
|
||||||
var ok bool
|
|
||||||
if val, ok = cpuStateAlertValue(name, data.Stats.CpuBreakdown); !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
triggered := alertData.Triggered
|
triggered := alertData.Triggered
|
||||||
@@ -254,16 +208,6 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
|
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// add zfs pool usage from historical record
|
|
||||||
for key, pool := range stats.ZfsPools {
|
|
||||||
if !pool.Raw && pool.Total > 0 {
|
|
||||||
zfsKey := zfsDiskAlertKey(key)
|
|
||||||
if _, ok := alert.mapSums[zfsKey]; !ok {
|
|
||||||
alert.mapSums[zfsKey] = 0.0
|
|
||||||
}
|
|
||||||
alert.mapSums[zfsKey] += float32(pool.Used / pool.Total * 100)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case "Temperature":
|
case "Temperature":
|
||||||
if alert.mapSums == nil {
|
if alert.mapSums == nil {
|
||||||
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
|
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
|
||||||
@@ -297,20 +241,13 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
}
|
}
|
||||||
alert.val += float64(stats.Battery[0])
|
alert.val += float64(stats.Battery[0])
|
||||||
default:
|
default:
|
||||||
value, ok := cpuStateAlertValue(alert.name, stats.CpuBreakdown)
|
continue
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
alert.val += value
|
|
||||||
}
|
}
|
||||||
alert.count++
|
alert.count++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// sum up vals for each alert
|
// sum up vals for each alert
|
||||||
for _, alert := range validAlerts {
|
for _, alert := range validAlerts {
|
||||||
if alert.count == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
switch alert.name {
|
switch alert.name {
|
||||||
case "Disk":
|
case "Disk":
|
||||||
maxPct := float32(0)
|
maxPct := float32(0)
|
||||||
@@ -318,12 +255,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
sumPct := float32(value)
|
sumPct := float32(value)
|
||||||
if sumPct > maxPct {
|
if sumPct > maxPct {
|
||||||
maxPct = sumPct
|
maxPct = sumPct
|
||||||
alert.descriptor = diskAlertDescriptor(key)
|
alert.descriptor = fmt.Sprintf("Usage of %s", key)
|
||||||
if poolKey, ok := strings.CutPrefix(key, "zfs:"); ok {
|
|
||||||
if pool := data.Stats.ZfsPools[poolKey]; pool != nil && pool.DisplayName != "" {
|
|
||||||
alert.descriptor = diskAlertDescriptor(zfsDiskAlertKey(pool.DisplayName))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
alert.val = float64(maxPct / float32(alert.count))
|
alert.val = float64(maxPct / float32(alert.count))
|
||||||
@@ -369,17 +301,6 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func zfsDiskAlertKey(poolName string) string {
|
|
||||||
return "zfs:" + poolName
|
|
||||||
}
|
|
||||||
|
|
||||||
func diskAlertDescriptor(key string) string {
|
|
||||||
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
|
|
||||||
return fmt.Sprintf("Usage of storage pool %s", poolName)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("Usage of %s", key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
|
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
|
||||||
return legacy != [2]uint8{} || len(batteries) > 0
|
return legacy != [2]uint8{} || len(batteries) > 0
|
||||||
}
|
}
|
||||||
@@ -388,9 +309,6 @@ func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
|
|||||||
// log.Printf("Sending alert %s: val %f | count %d | threshold %f\n", alert.name, alert.val, alert.count, alert.threshold)
|
// log.Printf("Sending alert %s: val %f | count %d | threshold %f\n", alert.name, alert.val, alert.count, alert.threshold)
|
||||||
systemName := alert.systemRecord.GetString("name")
|
systemName := alert.systemRecord.GetString("name")
|
||||||
|
|
||||||
if state, ok := cpuStateAlerts[alert.name]; ok {
|
|
||||||
alert.name = state.label
|
|
||||||
}
|
|
||||||
// change Disk to Disk usage
|
// change Disk to Disk usage
|
||||||
if alert.name == "Disk" {
|
if alert.name == "Disk" {
|
||||||
alert.name += " usage"
|
alert.name += " usage"
|
||||||
@@ -402,7 +320,7 @@ func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
|
|||||||
|
|
||||||
// make title alert name lowercase if not CPU or GPU
|
// make title alert name lowercase if not CPU or GPU
|
||||||
titleAlertName := alert.name
|
titleAlertName := alert.name
|
||||||
if titleAlertName != "CPU" && titleAlertName != "GPU" && !strings.HasPrefix(titleAlertName, "CPU") {
|
if titleAlertName != "CPU" && titleAlertName != "GPU" {
|
||||||
titleAlertName = strings.ToLower(titleAlertName)
|
titleAlertName = strings.ToLower(titleAlertName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -146,20 +146,6 @@ func setCPUAlertValue(info *system.Info, stats *system.Stats, value float64) {
|
|||||||
stats.Cpu = value
|
stats.Cpu = value
|
||||||
}
|
}
|
||||||
|
|
||||||
func setCPUStateAlertValue(_ *system.Info, stats *system.Stats, value []float64) {
|
|
||||||
stats.CpuBreakdown = value
|
|
||||||
}
|
|
||||||
|
|
||||||
var cpuStateAlertTests = []struct {
|
|
||||||
name string
|
|
||||||
trigger []float64
|
|
||||||
resolve []float64
|
|
||||||
baseline []float64
|
|
||||||
}{
|
|
||||||
{"CPUIOWait", []float64{0, 0, 51, 0, 49}, []float64{0, 0, 48, 0, 52}, []float64{0, 0, 10, 0, 90}},
|
|
||||||
{"CPUSteal", []float64{0, 0, 0, 51, 49}, []float64{0, 0, 0, 48, 52}, []float64{0, 0, 0, 10, 90}},
|
|
||||||
}
|
|
||||||
|
|
||||||
func setMemoryAlertValue(info *system.Info, stats *system.Stats, value float64) {
|
func setMemoryAlertValue(info *system.Info, stats *system.Stats, value float64) {
|
||||||
info.MemPct = value
|
info.MemPct = value
|
||||||
stats.MemPct = value
|
stats.MemPct = value
|
||||||
@@ -205,11 +191,6 @@ func setBatteryAlertValue(info *system.Info, stats *system.Stats, value [2]uint8
|
|||||||
|
|
||||||
func TestSystemAlertsOneMin(t *testing.T) {
|
func TestSystemAlertsOneMin(t *testing.T) {
|
||||||
testOneMinuteSystemAlert(t, "CPU", 50, setCPUAlertValue, 51, 49)
|
testOneMinuteSystemAlert(t, "CPU", 50, setCPUAlertValue, 51, 49)
|
||||||
for _, test := range cpuStateAlertTests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
testOneMinuteSystemAlert(t, test.name, 50, setCPUStateAlertValue, test.trigger, test.resolve)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
testOneMinuteSystemAlert(t, "Memory", 50, setMemoryAlertValue, 51, 49)
|
testOneMinuteSystemAlert(t, "Memory", 50, setMemoryAlertValue, 51, 49)
|
||||||
testOneMinuteSystemAlert(t, "Disk", 50, setDiskAlertValue, 51, 49)
|
testOneMinuteSystemAlert(t, "Disk", 50, setDiskAlertValue, 51, 49)
|
||||||
testOneMinuteSystemAlert(t, "Bandwidth", 50, setBandwidthAlertValue, [2]uint64{megabytesToBytes(26), megabytesToBytes(25)}, [2]uint64{megabytesToBytes(25), megabytesToBytes(24)})
|
testOneMinuteSystemAlert(t, "Bandwidth", 50, setBandwidthAlertValue, [2]uint64{megabytesToBytes(26), megabytesToBytes(25)}, [2]uint64{megabytesToBytes(25), megabytesToBytes(24)})
|
||||||
@@ -223,11 +204,6 @@ func TestSystemAlertsOneMin(t *testing.T) {
|
|||||||
|
|
||||||
func TestSystemAlertsTwoMin(t *testing.T) {
|
func TestSystemAlertsTwoMin(t *testing.T) {
|
||||||
testMultiMinuteSystemAlert(t, "CPU", 50, 2, setCPUAlertValue, 10, 51, 48)
|
testMultiMinuteSystemAlert(t, "CPU", 50, 2, setCPUAlertValue, 10, 51, 48)
|
||||||
for _, test := range cpuStateAlertTests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
testMultiMinuteSystemAlert(t, test.name, 50, 2, setCPUStateAlertValue, test.baseline, test.trigger, test.resolve)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
testMultiMinuteSystemAlert(t, "Memory", 50, 2, setMemoryAlertValue, 10, 51, 48)
|
testMultiMinuteSystemAlert(t, "Memory", 50, 2, setMemoryAlertValue, 10, 51, 48)
|
||||||
testMultiMinuteSystemAlert(t, "Disk", 50, 2, setDiskAlertValue, 10, 51, 48)
|
testMultiMinuteSystemAlert(t, "Disk", 50, 2, setDiskAlertValue, 10, 51, 48)
|
||||||
testMultiMinuteSystemAlert(t, "Bandwidth", 50, 2, setBandwidthAlertValue, [2]uint64{megabytesToBytes(10), megabytesToBytes(10)}, [2]uint64{megabytesToBytes(26), megabytesToBytes(25)}, [2]uint64{megabytesToBytes(10), megabytesToBytes(10)})
|
testMultiMinuteSystemAlert(t, "Bandwidth", 50, 2, setBandwidthAlertValue, [2]uint64{megabytesToBytes(10), megabytesToBytes(10)}, [2]uint64{megabytesToBytes(26), megabytesToBytes(25)}, [2]uint64{megabytesToBytes(10), megabytesToBytes(10)})
|
||||||
@@ -238,17 +214,3 @@ func TestSystemAlertsTwoMin(t *testing.T) {
|
|||||||
testMultiMinuteSystemAlert(t, "LoadAvg15", 4, 2, setLoadAvgAlertValue, [3]float64{0, 0, 2}, [3]float64{0, 0, 4.1}, [3]float64{0, 0, 3.5})
|
testMultiMinuteSystemAlert(t, "LoadAvg15", 4, 2, setLoadAvgAlertValue, [3]float64{0, 0, 2}, [3]float64{0, 0, 4.1}, [3]float64{0, 0, 3.5})
|
||||||
testMultiMinuteSystemAlert(t, "Battery", 20, 2, setBatteryAlertValue, [2]uint8{21, 0}, [2]uint8{19, 0}, [2]uint8{25, 1})
|
testMultiMinuteSystemAlert(t, "Battery", 20, 2, setBatteryAlertValue, [2]uint8{21, 0}, [2]uint8{19, 0}, [2]uint8{25, 1})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCPUStateAlertWithoutBreakdown(t *testing.T) {
|
|
||||||
fixture := newSystemAlertTestFixture(t, "CPUSteal", 1, 1)
|
|
||||||
defer fixture.cleanup()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
submitValue(fixture, t, []float64(nil), setCPUStateAlertValue)
|
|
||||||
submitValue(fixture, t, []float64{0, 0, 0, 0, 0}, setCPUStateAlertValue)
|
|
||||||
waitForSystemAlert(time.Second)
|
|
||||||
|
|
||||||
fixture.assertTriggered(t, false, "Alert should ignore missing CPU breakdown data")
|
|
||||||
assert.Zero(t, fixture.hub.TestMailer.TotalSend(), "No email should be sent without CPU breakdown data")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,190 +0,0 @@
|
|||||||
package alerts
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
)
|
|
||||||
|
|
||||||
// alertNameSystemdFailed is the alerts.name value for the failed systemd services alert.
|
|
||||||
const alertNameSystemdFailed = "SystemdFailed"
|
|
||||||
|
|
||||||
// maxListedServices caps how many service names are listed in a notification body.
|
|
||||||
const maxListedServices = 10
|
|
||||||
|
|
||||||
// HandleSystemdAlerts manages alerts for systemd services in the failed state.
|
|
||||||
//
|
|
||||||
// This is a binary state alert and fires on the first observation of a failed
|
|
||||||
// service rather than using a delay. The agent only refreshes systemd state every
|
|
||||||
// 10 minutes, so a shorter delay could never observe new data before expiring, and
|
|
||||||
// that poll interval already hides services that fail and restart quickly.
|
|
||||||
func (am *AlertManager) HandleSystemdAlerts(systemRecord *core.Record) error {
|
|
||||||
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameSystemdFailed)
|
|
||||||
if len(alerts) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// State is read from the systemd_services snapshot rather than the update payload.
|
|
||||||
// The payload is not a reliable source here: realtime dashboard subscriptions fetch
|
|
||||||
// from the agent with a shorter cache time, and the agent omits systemd services from
|
|
||||||
// those responses, overwriting the cached payload roughly once a second while a system
|
|
||||||
// is being viewed. The snapshot table is only written by the full update cycle.
|
|
||||||
total, failed, err := am.queryServiceStates(systemRecord.Id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if total == 0 {
|
|
||||||
// No rows normally means no systemd data for this system (agent without
|
|
||||||
// systemd, or not yet reported), which must not be treated as a recovery.
|
|
||||||
// Read info only in this ambiguous case. The record being saved is used
|
|
||||||
// instead of data because dashboard polling can replace the system's
|
|
||||||
// in-memory payload concurrently.
|
|
||||||
var currentInfo system.Info
|
|
||||||
if err := systemRecord.UnmarshalJSONField("info", ¤tInfo); err != nil ||
|
|
||||||
len(currentInfo.Services) == 0 || currentInfo.Services[0] != 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
systemName := systemRecord.GetString("name")
|
|
||||||
|
|
||||||
for _, alertData := range alerts {
|
|
||||||
triggered := len(failed) > 0
|
|
||||||
// Only notify on a change of state, so a service that stays failed across
|
|
||||||
// cycles doesn't re-notify every update.
|
|
||||||
if triggered == alertData.Triggered {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := am.sendSystemdAlert(triggered, systemName, alertData, failed); err != nil {
|
|
||||||
am.hub.Logger().Error("Failed to send alert", "err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// queryServiceStates returns the number of services reported in the most recent update
|
|
||||||
// for a system, and the names of those in the failed state.
|
|
||||||
//
|
|
||||||
// Rows are restricted to the latest update because systemd_services is upserted, never
|
|
||||||
// pruned on change: a service that no longer exists on the host stops being reported and
|
|
||||||
// its row keeps its last known state until the retention sweep removes it. Every row
|
|
||||||
// written in one cycle shares a single updated timestamp, so the newest timestamp
|
|
||||||
// identifies exactly the services the agent last reported.
|
|
||||||
func (am *AlertManager) queryServiceStates(systemID string) (total int, failed []string, err error) {
|
|
||||||
var rows []struct {
|
|
||||||
Name string `db:"name"`
|
|
||||||
State systemd.ServiceState `db:"state"`
|
|
||||||
}
|
|
||||||
err = am.hub.DB().
|
|
||||||
Select("name", "state").
|
|
||||||
From("systemd_services").
|
|
||||||
Where(dbx.NewExp(
|
|
||||||
"system={:system} AND updated=(SELECT MAX(updated) FROM systemd_services WHERE system={:system})",
|
|
||||||
dbx.Params{"system": systemID},
|
|
||||||
)).
|
|
||||||
OrderBy("name").
|
|
||||||
All(&rows)
|
|
||||||
if err != nil {
|
|
||||||
return 0, nil, err
|
|
||||||
}
|
|
||||||
for _, row := range rows {
|
|
||||||
if row.State == systemd.StatusFailed {
|
|
||||||
failed = append(failed, row.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return len(rows), failed, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendSystemdAlert sends a failed or recovered systemd services alert to the alert's user.
|
|
||||||
func (am *AlertManager) sendSystemdAlert(triggered bool, systemName string, alertData CachedAlertData, failed []string) error {
|
|
||||||
// Update trigger state for alert record before sending alert
|
|
||||||
if err := am.setAlertTriggered(alertData, triggered); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var title, message string
|
|
||||||
if triggered {
|
|
||||||
title = fmt.Sprintf("Failed services on %s %v", systemName, "\U0001F534") // Red alert emoji
|
|
||||||
message = fmt.Sprintf("%s on %s: %s", pluralizeServices(len(failed)), systemName, formatServiceList(failed))
|
|
||||||
} else {
|
|
||||||
title = fmt.Sprintf("Services recovered on %s %v", systemName, "✅") // Green checkmark emoji
|
|
||||||
message = fmt.Sprintf("No services are in the failed state on %s.", systemName)
|
|
||||||
}
|
|
||||||
|
|
||||||
systemID := alertData.SystemID
|
|
||||||
|
|
||||||
return am.SendAlert(AlertMessageData{
|
|
||||||
UserID: alertData.UserID,
|
|
||||||
SystemID: systemID,
|
|
||||||
Title: title,
|
|
||||||
Message: message,
|
|
||||||
Link: am.hub.MakeLink("system", systemID),
|
|
||||||
LinkText: "View " + systemName,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// pluralizeServices returns a count label like "1 failed service" or "3 failed services".
|
|
||||||
func pluralizeServices(count int) string {
|
|
||||||
if count == 1 {
|
|
||||||
return "1 failed service"
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%d failed services", count)
|
|
||||||
}
|
|
||||||
|
|
||||||
// formatServiceList joins service names, truncating long lists.
|
|
||||||
func formatServiceList(names []string) string {
|
|
||||||
if len(names) <= maxListedServices {
|
|
||||||
return strings.Join(names, ", ")
|
|
||||||
}
|
|
||||||
remaining := len(names) - maxListedServices
|
|
||||||
return fmt.Sprintf("%s and %d more", strings.Join(names[:maxListedServices], ", "), remaining)
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveSystemdAlerts resolves triggered systemd alerts for systems that no longer
|
|
||||||
// have any failed services. This clears stale state left by a hub restart.
|
|
||||||
func resolveSystemdAlerts(app core.App) error {
|
|
||||||
db := app.DB()
|
|
||||||
var alertIds []string
|
|
||||||
err := db.NewQuery(`
|
|
||||||
SELECT a.id
|
|
||||||
FROM alerts a
|
|
||||||
JOIN systems sys ON sys.id = a.system
|
|
||||||
WHERE a.name = {:name}
|
|
||||||
AND a.triggered = true
|
|
||||||
AND (
|
|
||||||
EXISTS (
|
|
||||||
SELECT 1 FROM systemd_services cur
|
|
||||||
WHERE cur.system = a.system
|
|
||||||
AND cur.updated = (SELECT MAX(updated) FROM systemd_services WHERE system = a.system)
|
|
||||||
)
|
|
||||||
OR json_extract(sys.info, '$.sv[0]') = 0
|
|
||||||
)
|
|
||||||
AND NOT EXISTS (
|
|
||||||
SELECT 1 FROM systemd_services s
|
|
||||||
WHERE s.system = a.system AND s.state = {:state}
|
|
||||||
AND s.updated = (SELECT MAX(updated) FROM systemd_services WHERE system = a.system)
|
|
||||||
)
|
|
||||||
`).Bind(dbx.Params{
|
|
||||||
"name": alertNameSystemdFailed,
|
|
||||||
"state": systemd.StatusFailed,
|
|
||||||
}).Column(&alertIds)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, alertId := range alertIds {
|
|
||||||
alert, err := app.FindRecordById("alerts", alertId)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
alert.Set("triggered", false)
|
|
||||||
if err := app.Save(alert); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,383 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package alerts_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/alerts"
|
|
||||||
systemEntity "github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// setSystemdServiceState upserts a systemd_services row mirroring the raw SQL write
|
|
||||||
// path used by the hub (createSystemdStatsRecords), which bypasses record hooks.
|
|
||||||
func setSystemdServiceState(t *testing.T, hub core.App, systemID, name string, state systemd.ServiceState, updated int64) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := hub.DB().NewQuery(
|
|
||||||
"INSERT INTO systemd_services (id, system, name, state, sub, cpu, cpuPeak, memory, memPeak, updated) " +
|
|
||||||
"VALUES ({:id}, {:system}, {:name}, {:state}, 0, 0, 0, 0, 0, {:updated}) " +
|
|
||||||
"ON CONFLICT(id) DO UPDATE SET state = excluded.state, updated = excluded.updated",
|
|
||||||
).Bind(dbx.Params{
|
|
||||||
"id": systemID + "-" + name,
|
|
||||||
"system": systemID,
|
|
||||||
"name": name,
|
|
||||||
"state": state,
|
|
||||||
"updated": updated,
|
|
||||||
}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedServices writes a set of services into the systemd_services snapshot, which is the
|
|
||||||
// source HandleSystemdAlerts reads from. All rows share one updated timestamp, matching
|
|
||||||
// how the hub writes a batch in createSystemdStatsRecords.
|
|
||||||
func seedServices(t *testing.T, hub core.App, systemID string, states ...systemd.ServiceState) {
|
|
||||||
t.Helper()
|
|
||||||
seedServicesAt(t, hub, systemID, time.Now().UTC().UnixMilli(), states...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedServicesAt writes services with an explicit batch timestamp.
|
|
||||||
func seedServicesAt(t *testing.T, hub core.App, systemID string, updated int64, states ...systemd.ServiceState) {
|
|
||||||
t.Helper()
|
|
||||||
for i, state := range states {
|
|
||||||
setSystemdServiceState(t, hub, systemID, serviceName(i), state, updated)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func serviceName(i int) string {
|
|
||||||
return string(rune('a'+i)) + ".service"
|
|
||||||
}
|
|
||||||
|
|
||||||
// systemdTestSetup creates a user with an email, a system, and a SystemdFailed alert.
|
|
||||||
func systemdTestSetup(t *testing.T, triggered bool) (*beszelTests.TestHub, *core.Record, *core.Record) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
|
||||||
|
|
||||||
userSettings, err := hub.FindFirstRecordByFilter("user_settings", "user={:user}", map[string]any{"user": user.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
userSettings.Set("settings", `{"emails":["test@example.com"],"webhooks":[]}`)
|
|
||||||
require.NoError(t, hub.Save(userSettings))
|
|
||||||
|
|
||||||
// "paused" avoids spawning a background updater goroutine that would outlive
|
|
||||||
// the test hub; these tests drive HandleSystemdAlerts directly.
|
|
||||||
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused")
|
|
||||||
require.NoError(t, err)
|
|
||||||
system := systems[0]
|
|
||||||
|
|
||||||
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
|
||||||
"name": "SystemdFailed",
|
|
||||||
"system": system.Id,
|
|
||||||
"user": user.Id,
|
|
||||||
"triggered": triggered,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return hub, system, alert
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertFiresImmediately(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, false)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusFailed, systemd.StatusActive)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "failed service should notify on first observation")
|
|
||||||
|
|
||||||
messages := hub.TestMailer.Messages()
|
|
||||||
require.NotEmpty(t, messages)
|
|
||||||
last := messages[len(messages)-1]
|
|
||||||
assert.Contains(t, last.Subject, "Failed services")
|
|
||||||
assert.Contains(t, last.Text, "a.service", "notification should name the failed service")
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, alertRecord.GetBool("triggered"), "alert should be marked triggered")
|
|
||||||
|
|
||||||
// history record should be created via the alerts update hook
|
|
||||||
historyCount, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.EqualValues(t, 1, historyCount, "should have one unresolved alert history record")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertFullCycle(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, false)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
// Fail, then recover.
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusFailed)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusActive)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount+2, hub.TestMailer.TotalSend(), "should send a failure and a recovery notification")
|
|
||||||
|
|
||||||
messages := hub.TestMailer.Messages()
|
|
||||||
require.Len(t, messages, 2)
|
|
||||||
assert.Contains(t, messages[0].Subject, "Failed services")
|
|
||||||
assert.Contains(t, messages[1].Subject, "Services recovered")
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "alert should be cleared after recovery")
|
|
||||||
|
|
||||||
// history record should be resolved
|
|
||||||
historyCount, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Zero(t, historyCount, "alert history record should be resolved")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertSendsRecoveryWhenTriggered(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusActive, systemd.StatusInactive)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "recovery notification should be sent")
|
|
||||||
messages := hub.TestMailer.Messages()
|
|
||||||
require.NotEmpty(t, messages)
|
|
||||||
assert.Contains(t, messages[len(messages)-1].Subject, "Services recovered")
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "alert should be cleared after recovery")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertDoesNotResendWhileTriggered(t *testing.T) {
|
|
||||||
hub, system, _ := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
// Still failing across several cycles — should not re-notify.
|
|
||||||
for range 3 {
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusFailed)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount, hub.TestMailer.TotalSend(), "should not re-notify while still triggered")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertRepeatedFailureNotifiesOnce(t *testing.T) {
|
|
||||||
hub, system, _ := systemdTestSetup(t, false)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
for range 3 {
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusFailed)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "repeated failures should only notify once")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A service that no longer exists on the host stops being reported, but its row stays
|
|
||||||
// in systemd_services with its last known state until the retention sweep. That stale
|
|
||||||
// row must not keep the alert triggered.
|
|
||||||
func TestSystemdAlertIgnoresServicesNoLongerReported(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
now := time.Now().UTC().UnixMilli()
|
|
||||||
// Older batch still holding a failed service that has since been removed.
|
|
||||||
setSystemdServiceState(t, hub, system.Id, "gone.service", systemd.StatusFailed, now-60_000)
|
|
||||||
// Current batch reports only healthy services.
|
|
||||||
seedServicesAt(t, hub, system.Id, now, systemd.StatusActive, systemd.StatusActive)
|
|
||||||
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "stale failed row should not block recovery")
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "alert should resolve once the service stops being reported")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveSystemdAlertsIgnoresStaleFailedRows(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
now := time.Now().UTC().UnixMilli()
|
|
||||||
setSystemdServiceState(t, hub, system.Id, "gone.service", systemd.StatusFailed, now-60_000)
|
|
||||||
seedServicesAt(t, hub, system.Id, now, systemd.StatusActive)
|
|
||||||
|
|
||||||
require.NoError(t, alerts.ResolveSystemdAlerts(hub))
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "stale failed row should not keep the alert triggered")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertNoSystemdDataIsIgnored(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
// A system with no systemd_services rows (agent without systemd, or nothing
|
|
||||||
// reported yet) must not be treated as a recovery.
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount, hub.TestMailer.TotalSend(), "missing systemd data should not send a recovery")
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, alertRecord.GetBool("triggered"), "triggered state should be preserved when data is absent")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertFreshEmptySnapshotResolves(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
// An explicit zero service count on the saved system record distinguishes a
|
|
||||||
// confirmed empty snapshot from an agent response that omitted systemd data.
|
|
||||||
system.Set("info", systemEntity.Info{Services: []uint16{0, 0}})
|
|
||||||
require.NoError(t, am.HandleSystemAlerts(system, nil))
|
|
||||||
|
|
||||||
assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "fresh empty snapshot should send a recovery")
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "fresh empty snapshot should resolve the alert")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertNoAlertRecord(t *testing.T) {
|
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused")
|
|
||||||
require.NoError(t, err)
|
|
||||||
system := systems[0]
|
|
||||||
|
|
||||||
initialEmailCount := hub.TestMailer.TotalSend()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusFailed)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
assert.Equal(t, initialEmailCount, hub.TestMailer.TotalSend(), "no email when no alert record exists")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveSystemdAlertsClearsStaleTriggered(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
// No failed services in the snapshot, but the alert is still marked triggered
|
|
||||||
// (e.g. the hub restarted while the alert was active).
|
|
||||||
setSystemdServiceState(t, hub, system.Id, "a.service", systemd.StatusActive, time.Now().UTC().UnixMilli())
|
|
||||||
|
|
||||||
require.NoError(t, alerts.ResolveSystemdAlerts(hub))
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "stale triggered flag should be cleared")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveSystemdAlertsKeepsTriggeredWithoutSystemdData(t *testing.T) {
|
|
||||||
hub, _, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
// Missing rows do not prove recovery. This can happen when a system is offline
|
|
||||||
// and its last service snapshot has been removed by retention.
|
|
||||||
require.NoError(t, alerts.ResolveSystemdAlerts(hub))
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, alertRecord.GetBool("triggered"), "missing systemd data should preserve triggered state")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveSystemdAlertsClearsConfirmedEmptySnapshot(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
// Update the persisted snapshot directly so record hooks don't alter alert state
|
|
||||||
// before the startup resolver is exercised.
|
|
||||||
_, err := hub.DB().NewQuery(
|
|
||||||
"UPDATE systems SET info = {:info} WHERE id = {:id}",
|
|
||||||
).Bind(dbx.Params{"info": `{"sv":[0,0]}`, "id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, alerts.ResolveSystemdAlerts(hub))
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alertRecord.GetBool("triggered"), "confirmed empty snapshot should clear triggered state")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveSystemdAlertsKeepsStillFailing(t *testing.T) {
|
|
||||||
hub, system, alert := systemdTestSetup(t, true)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
setSystemdServiceState(t, hub, system.Id, "a.service", systemd.StatusFailed, time.Now().UTC().UnixMilli())
|
|
||||||
|
|
||||||
require.NoError(t, alerts.ResolveSystemdAlerts(hub))
|
|
||||||
|
|
||||||
alertRecord, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, alertRecord.GetBool("triggered"), "alert should stay triggered while a service is still failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemdAlertMultipleUsersRespectOwnAlerts(t *testing.T) {
|
|
||||||
hub, user1 := beszelTests.GetHubWithUser(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
setStatusAlertEmail(t, hub, user1.Id, "user1@example.com")
|
|
||||||
|
|
||||||
user2, err := beszelTests.CreateUser(hub, "user2@example.com", "password")
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, err = beszelTests.CreateRecord(hub, "user_settings", map[string]any{
|
|
||||||
"user": user2.Id,
|
|
||||||
"settings": map[string]any{
|
|
||||||
"emails": []string{"user2@example.com"},
|
|
||||||
"webhooks": []string{},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
|
||||||
"name": "shared-system",
|
|
||||||
"users": []string{user1.Id, user2.Id},
|
|
||||||
"host": "127.0.0.1",
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, user := range []*core.Record{user1, user2} {
|
|
||||||
_, err = beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
|
||||||
"name": "SystemdFailed",
|
|
||||||
"system": system.Id,
|
|
||||||
"user": user.Id,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
seedServices(t, hub, system.Id, systemd.StatusFailed)
|
|
||||||
require.NoError(t, am.HandleSystemdAlerts(system))
|
|
||||||
|
|
||||||
messages := hub.TestMailer.Messages()
|
|
||||||
require.Len(t, messages, 2, "each user should receive their own alert")
|
|
||||||
}
|
|
||||||
@@ -11,9 +11,8 @@ import (
|
|||||||
|
|
||||||
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
|
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
|
||||||
return &AlertManager{
|
return &AlertManager{
|
||||||
hub: app,
|
hub: app,
|
||||||
alertsCache: NewAlertsCache(app),
|
alertsCache: NewAlertsCache(app),
|
||||||
networkMonitors: newNetworkMonitorCache(app),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -89,10 +88,6 @@ func ResolveStatusAlerts(app core.App) error {
|
|||||||
return resolveStatusAlerts(app)
|
return resolveStatusAlerts(app)
|
||||||
}
|
}
|
||||||
|
|
||||||
func ResolveSystemdAlerts(app core.App) error {
|
|
||||||
return resolveSystemdAlerts(app)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) RestorePendingStatusAlerts() error {
|
func (am *AlertManager) RestorePendingStatusAlerts() error {
|
||||||
return am.restorePendingStatusAlerts()
|
return am.restorePendingStatusAlerts()
|
||||||
}
|
}
|
||||||
@@ -101,7 +96,6 @@ func (am *AlertManager) SetAlertTriggered(alert CachedAlertData, triggered bool)
|
|||||||
return am.setAlertTriggered(alert, triggered)
|
return am.setAlertTriggered(alert, triggered)
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildContainerLogExcerpt exposes buildContainerLogExcerpt for testing.
|
func IsInternalURL(rawURL string) (bool, error) {
|
||||||
func BuildContainerLogExcerpt(raw string) string {
|
return isInternalURL(rawURL)
|
||||||
return buildContainerLogExcerpt(raw)
|
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user