Compare commits

..
Author SHA1 Message Date
Chris Lu a4bf9fe47f test(remote_storage/s3): assert WriteFile success in mime tests 2026-05-27 12:15:21 -07:00
Chris Lu b3ea57b5d7 fix(remote_storage/s3): forward entry.Attributes.Mime as ContentType
filer.remote.sync was uploading every object without a Content-Type, so
S3-compatible backends (e.g. Backblaze B2) stored binary/octet-stream
and browsers refused to render HTML, CSS, etc.

Pass entry.Attributes.Mime through to UploadInput.ContentType, leaving
the header unset when no Mime is recorded so the remote keeps its own
default behavior.
2026-05-27 10:34:11 -07:00
260 changed files with 10556 additions and 11703 deletions
+59 -19
View File
@@ -118,9 +118,8 @@ jobs:
needs: [build-rust-binaries]
runs-on: ubuntu-latest
strategy:
# All variants at once: pulls hit mirror.gcr.io and pushes go to GHCR only,
# so docker.io limits don't apply. Watch the 10 GB gha cache budget.
max-parallel: 5
# Build sequentially to avoid rate limits
max-parallel: 2
matrix:
include:
# Normal volume - multi-arch
@@ -273,52 +272,93 @@ jobs:
BRANCH=${{ github.sha }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
# Copy GHCR -> Docker Hub here, per variant, so it overlaps with the other
# variants still building instead of waiting on the whole matrix.
- name: Clean up build artifacts
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
run: |
sudo docker system prune -f
sudo rm -rf /tmp/go-build*
copy-to-dockerhub:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
strategy:
matrix:
variant: [normal, large_disk, full, large_disk_full, rocksdb]
include:
- variant: normal
tag_suffix: ""
- variant: large_disk
tag_suffix: _large_disk
- variant: full
tag_suffix: _full
- variant: large_disk_full
tag_suffix: _large_disk_full
- variant: rocksdb
tag_suffix: _large_disk_rocksdb
steps:
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.2.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.2.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Copy ${{ matrix.variant }} to Docker Hub
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
# Function to retry with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Copy multi-arch image from GHCR to Docker Hub with retry
# This is much more efficient than pulling/pushing individual arch images
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
retry_with_backoff crane copy \
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
echo "Copied ${{ matrix.variant }} to Docker Hub"
- name: Clean up build artifacts
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
run: |
sudo docker system prune -f
sudo rm -rf /tmp/go-build*
echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
# Security for visibility, but never blocks the release. Releases (including
@@ -378,7 +418,7 @@ jobs:
# trivy-scan: vuln findings are reported but do not block `latest`.
tag-latest:
runs-on: ubuntu-latest
needs: [build]
needs: [copy-to-dockerhub]
if: github.event_name == 'push'
strategy:
matrix:
@@ -443,7 +483,7 @@ jobs:
helm-release:
runs-on: ubuntu-latest
needs: [build]
needs: [copy-to-dockerhub]
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
permissions:
contents: write
+137
View File
@@ -0,0 +1,137 @@
name: "NFS Integration Tests"
on:
push:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
concurrency:
group: ${{ github.head_ref }}/nfs-tests
cancel-in-progress: true
permissions:
contents: read
env:
TEST_TIMEOUT: '15m'
jobs:
nfs-integration:
name: NFS Integration Testing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Run NFS Integration Tests
run: |
cd test/nfs
echo "Running NFS integration tests..."
echo "============================================"
# Install test dependencies
go mod download
# Run the protocol-layer tests. The kernel-mount tests require root
# for mount(2) and are exercised in their own privileged step below;
# skip them here so a "skipped because not root" line doesn't show
# up as noise on every CI run.
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
echo "============================================"
echo "NFS integration tests completed"
- name: Install kernel NFS client
run: |
# nfs-common provides mount.nfs; netbase provides /etc/protocols
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
sudo apt-get update
sudo apt-get install -y nfs-common netbase
- name: Run kernel-mount E2E tests
run: |
cd test/nfs
echo "Running kernel-mount end-to-end tests..."
echo "These mount the running 'weed nfs' subprocess via the actual"
echo "Linux NFS client to catch protocol regressions invisible to"
echo "the go-nfs-client-based tests above."
echo "============================================"
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
# binary that test/nfs/framework.go locates via $PATH) resolve
# correctly under sudo, and pass through the Go module/cache dirs
# so we don't redownload modules under root.
sudo env "PATH=$PATH" \
GOMODCACHE="$(go env GOMODCACHE)" \
GOCACHE="$(go env GOCACHE)" \
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
echo "============================================"
echo "Kernel-mount E2E tests completed"
- name: Test Summary
if: always()
run: |
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Harness" >> $GITHUB_STEP_SUMMARY
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
-80
View File
@@ -1,80 +0,0 @@
name: "terraform: validate and test modules"
on:
push:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
pull_request:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
permissions:
contents: read
jobs:
validate:
name: fmt, validate, plan-level tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- name: Set up OpenTofu
uses: opentofu/setup-opentofu@v1
with:
tofu_version: 1.12.1
- name: fmt check
working-directory: terraform
run: tofu fmt -recursive -check -diff
- name: validate core
working-directory: terraform/modules/core
run: |
tofu init -backend=false -input=false
tofu validate
- name: validate security
working-directory: terraform/modules/security
run: |
tofu init -backend=false -input=false
tofu validate
- name: plan-level tests (core)
working-directory: terraform/modules/core
run: tofu test
- name: validate examples
run: |
set -e
for ex in terraform/examples/*/; do
echo "== validate $ex =="
tofu -chdir="$ex" init -backend=false -input=false
tofu -chdir="$ex" validate
done
smoke:
name: local cluster smoke test (real weed)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod
- name: Build weed
run: go build -o "$RUNNER_TEMP/weed" ./weed
- name: Set up OpenTofu
uses: opentofu/setup-opentofu@v1
with:
tofu_version: 1.12.1
- name: Run local cluster harness
working-directory: terraform/test/local
run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
- name: Run local mTLS cluster harness
working-directory: terraform/test/local-secure
run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh
+1 -3
View File
@@ -35,7 +35,6 @@ Your support will be really appreciated by me and other supporters!
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
[![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca)
[![zyner](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_zyner.png)](https://zyner.org)
---
@@ -655,8 +654,7 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
## Enterprise ##
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has advanced features, including data recovery, self-healing storage,
customizable erasure coding, EC vacuum and repair, etc.
which has a self-healing storage format with better data protection.
[Back to TOC](#table-of-contents)
+2 -7
View File
@@ -1,6 +1,4 @@
# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary,
# so arm64/arm/386 targets skip QEMU emulation of the whole compile.
FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder
FROM golang:1.25-alpine AS builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
ARG BRANCH=${BRANCH:-master}
@@ -14,12 +12,9 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
git checkout $BRANCH) || \
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
echo "Available branches:" && git branch -a && exit 1))
ARG TARGETOS TARGETARCH TARGETVARIANT
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& export GOOS=$TARGETOS GOARCH=$TARGETARCH \
&& case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \
&& CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed .
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
# Rust volume server: use pre-built binary from CI when available (placed in
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
-3
View File
@@ -127,9 +127,6 @@ test_tarantool: tags = tarantool
test_tarantool: build_tarantool_dev_env build
docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
test_keycloak_s3: build
docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
clean:
rm ./weed
-46
View File
@@ -1,46 +0,0 @@
services:
keycloak:
image: quay.io/keycloak/keycloak:26.0.7
command: ["start-dev", "--import-realm"]
environment:
KC_BOOTSTRAP_ADMIN_USERNAME: admin
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
KC_HTTP_ENABLED: "true"
KC_HOSTNAME: "http://keycloak:8080"
KC_HOSTNAME_STRICT: "false"
ports:
- "8080:8080"
volumes:
- ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
healthcheck:
test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
interval: 10s
timeout: 5s
retries: 5
start_period: 60s
s3:
image: chrislusf/seaweedfs:local
command:
- -v=9
- server
- -ip=s3
- -filer
- -master.volumeSizeLimitMB=16
- -volume
- -volume.max=0
- -volume.preStopSeconds=1
- -s3
- -s3.port=8333
- -s3.config=/etc/seaweedfs/s3_config.json
- -s3.iam.config=/etc/seaweedfs/iam_config.json
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volumes:
- ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
- ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
ports:
- "8333:8333"
depends_on:
keycloak:
condition: service_healthy
+4 -1
View File
@@ -129,6 +129,7 @@ require (
github.com/cognusion/imaging v1.0.3
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.44.1
github.com/go-git/go-billy/v5 v5.9.0
github.com/go-ldap/ldap/v3 v3.4.13
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -151,6 +152,8 @@ require (
github.com/tarantool/go-tarantool/v2 v2.4.2
github.com/testcontainers/testcontainers-go v0.40.0
github.com/tikv/client-go/v2 v2.0.7
github.com/willscott/go-nfs v0.0.4
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
@@ -208,7 +211,6 @@ require (
github.com/dromara/dongle v1.0.1 // indirect
github.com/gin-gonic/gin v1.11.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
github.com/go-git/go-billy/v5 v5.9.0 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/google/go-cmp v0.7.0 // indirect
@@ -255,6 +257,7 @@ require (
github.com/pquerna/otp v1.5.0 // indirect
github.com/pterm/pterm v0.12.82 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
github.com/rclone/go-proton-api v1.0.2 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
+6
View File
@@ -1783,6 +1783,8 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
@@ -2026,6 +2028,10 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.31"
appVersion: "4.29"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.31.0
version: 4.29.0
+1 -1
View File
@@ -384,4 +384,4 @@ helm install seaweedfs seaweedfs/seaweedfs \
## Enterprise
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.
which has a self-healing storage format with better data protection.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 746 B

+53 -164
View File
@@ -23,27 +23,6 @@ use crate::pb::volume_server_pb;
use crate::storage::needle::needle::Needle;
use crate::storage::types::*;
/// Slack added over the configured file-size limit when bounding the raw
/// request body, to allow for multipart/form-data framing overhead. The exact
/// per-file limit is still enforced on the parsed data after multipart parsing.
const UPLOAD_BODY_OVERHEAD: usize = 16 * 1024 * 1024; // 16 MiB
/// Upper bound on bytes we will materialize in memory for a single request when
/// expanding stored content (gzip decompression or chunk-manifest assembly).
/// Guards against gzip bombs and crafted/oversized manifest sizes OOM-killing
/// the server; legitimate large objects are read via client-side chunk fetches.
const MAX_EXPANSION_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
/// Why `maybe_decompress_gzip` failed, so callers can distinguish a recoverable
/// "not valid gzip, use the raw bytes" from "the bomb cap was hit, reject it".
#[derive(Debug)]
enum GunzipError {
/// Input was not valid gzip / decode failed — callers may fall back to raw.
Decode,
/// Decompressed output would exceed `MAX_EXPANSION_BYTES` — must be rejected.
TooLarge,
}
// ============================================================================
// Inflight Throttle Guard
// ============================================================================
@@ -1473,16 +1452,12 @@ async fn get_or_head_handler_inner(
if is_compressed {
if needs_image_ops {
// Always decompress for image operations (Go decompresses before resize/crop)
match maybe_decompress_gzip(&data) {
Ok(decompressed) => data = decompressed,
Err(GunzipError::TooLarge) => {
return (
StatusCode::PAYLOAD_TOO_LARGE,
"compressed object exceeds decompression limit",
)
.into_response()
}
Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
use flate2::read::GzDecoder;
use std::io::Read as _;
let mut decoder = GzDecoder::new(&data[..]);
let mut decompressed = Vec::new();
if decoder.read_to_end(&mut decompressed).is_ok() {
data = decompressed;
}
} else {
let accept_encoding = headers
@@ -1499,16 +1474,12 @@ async fn get_or_head_handler_inner(
response_headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
} else {
// Decompress for client
match maybe_decompress_gzip(&data) {
Ok(decompressed) => data = decompressed,
Err(GunzipError::TooLarge) => {
return (
StatusCode::PAYLOAD_TOO_LARGE,
"compressed object exceeds decompression limit",
)
.into_response()
}
Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
use flate2::read::GzDecoder;
use std::io::Read as _;
let mut decoder = GzDecoder::new(&data[..]);
let mut decompressed = Vec::new();
if decoder.read_to_end(&mut decompressed).is_ok() {
data = decompressed;
}
}
}
@@ -2163,30 +2134,15 @@ pub async fn post_handler(
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
// Read body, bounded by the configured file-size limit so a single upload
// cannot buffer unbounded memory and OOM-kill the server (mirrors Go's
// io.LimitReader(r.Body, sizeLimit+1)). A margin covers multipart framing;
// the exact per-file limit is still enforced on the parsed data below.
let body_limit = if state.file_size_limit_bytes > 0 {
// try_from (not `as usize`) so a >usize::MAX limit on 32-bit caps at
// usize::MAX instead of silently truncating/wrapping to a tiny value.
usize::try_from(state.file_size_limit_bytes)
.unwrap_or(usize::MAX)
.saturating_add(UPLOAD_BODY_OVERHEAD)
} else {
usize::MAX
};
let body = match axum::body::to_bytes(request.into_body(), body_limit).await {
// Read body
let body = match axum::body::to_bytes(request.into_body(), usize::MAX).await {
Ok(b) => b,
Err(e) => {
// With a limit configured, an error here means the body exceeded it
// before we buffered the whole thing; report it like the size check.
let msg = if state.file_size_limit_bytes > 0 {
format!("file over the limited {} bytes", state.file_size_limit_bytes)
} else {
format!("read body: {}", e)
};
return json_error_with_query(StatusCode::BAD_REQUEST, msg, Some(&query));
return json_error_with_query(
StatusCode::BAD_REQUEST,
format!("read body: {}", e),
Some(&query),
)
}
};
@@ -2326,17 +2282,7 @@ pub async fn post_handler(
};
let uncompressed_data = if is_gzipped {
match maybe_decompress_gzip(&body_data_raw) {
Ok(d) => d,
Err(GunzipError::TooLarge) => {
return json_error_with_query(
StatusCode::PAYLOAD_TOO_LARGE,
"compressed object exceeds decompression limit",
Some(&query),
);
}
Err(GunzipError::Decode) => body_data_raw.clone(),
}
maybe_decompress_gzip(&body_data_raw).unwrap_or_else(|| body_data_raw.clone())
} else {
body_data_raw.clone()
};
@@ -2815,16 +2761,14 @@ pub async fn delete_handler(
// If this is a chunk manifest, delete child chunks first
if n.is_chunk_manifest() {
let manifest_data = if n.is_compressed() {
match maybe_decompress_gzip(&n.data) {
Ok(d) => d,
Err(GunzipError::TooLarge) => {
return json_error_with_query(
StatusCode::PAYLOAD_TOO_LARGE,
"compressed manifest exceeds decompression limit",
Some(&del_query),
);
}
Err(GunzipError::Decode) => n.data.clone(),
use flate2::read::GzDecoder;
use std::io::Read as _;
let mut decoder = GzDecoder::new(&n.data[..]);
let mut decompressed = Vec::new();
if decoder.read_to_end(&mut decompressed).is_ok() {
decompressed
} else {
n.data.clone()
}
} else {
n.data.clone()
@@ -3180,19 +3124,14 @@ fn try_expand_chunk_manifest(
last_modified_str: &Option<String>,
) -> Option<Response> {
let data = if n.is_compressed() {
match maybe_decompress_gzip(&n.data) {
Ok(d) => d,
Err(GunzipError::TooLarge) => {
return Some(
(
StatusCode::PAYLOAD_TOO_LARGE,
"compressed manifest exceeds decompression limit",
)
.into_response(),
)
}
Err(GunzipError::Decode) => return None,
use flate2::read::GzDecoder;
use std::io::Read as _;
let mut decoder = GzDecoder::new(&n.data[..]);
let mut decompressed = Vec::new();
if decoder.read_to_end(&mut decompressed).is_err() {
return None;
}
decompressed
} else {
n.data.clone()
};
@@ -3202,13 +3141,6 @@ fn try_expand_chunk_manifest(
Err(_) => return None,
};
// Guard the attacker-controlled manifest size before allocating: a negative
// value would wrap to a huge usize (capacity-overflow panic) and an oversized
// one would OOM-kill the server.
if manifest.size < 0 || manifest.size as u64 > MAX_EXPANSION_BYTES {
return None;
}
// Read and concatenate all chunks
let mut result = vec![0u8; manifest.size as usize];
let store = state.store.read().unwrap();
@@ -3242,54 +3174,24 @@ fn try_expand_chunk_manifest(
)
}
}
// Validate the attacker-controlled chunk offset before indexing: a
// negative value would wrap to a huge usize, and an out-of-range one has
// nowhere to land.
if chunk.offset < 0 {
return Some(
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("invalid negative chunk offset in {}", chunk.fid),
)
.into_response(),
);
}
let offset = chunk.offset as usize;
if offset >= result.len() {
continue;
}
// Write the chunk into its window in `result`. Compressed chunks inflate
// directly into the destination slice (bounded by the remaining window),
// so a chunk never allocates a second large buffer — peak memory stays at
// ~manifest.size instead of doubling. Bytes past the window are dropped,
// matching the prior truncation behavior.
if chunk_needle.is_compressed() {
let chunk_data = if chunk_needle.is_compressed() {
use flate2::read::GzDecoder;
use std::io::Read as _;
let mut decoder = GzDecoder::new(&chunk_needle.data[..]);
let window = &mut result[offset..];
let mut written = 0usize;
let mut decode_failed = false;
while written < window.len() {
match decoder.read(&mut window[written..]) {
Ok(0) => break,
Ok(n) => written += n,
Err(_) => {
decode_failed = true;
break;
}
}
}
// On any decode failure, drop the partial output and fall back to the
// chunk's raw bytes (truncated to the window), preserving prior behavior.
if decode_failed {
window[..written].fill(0);
let copy_len = chunk_needle.data.len().min(window.len());
window[..copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
let mut decompressed = Vec::new();
if decoder.read_to_end(&mut decompressed).is_ok() {
decompressed
} else {
chunk_needle.data.clone()
}
} else {
let copy_len = chunk_needle.data.len().min(result.len() - offset);
result[offset..offset + copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
chunk_needle.data.clone()
};
let offset = chunk.offset as usize;
let end = std::cmp::min(offset + chunk_data.len(), result.len());
let copy_len = end - offset;
if copy_len > 0 {
result[offset..offset + copy_len].copy_from_slice(&chunk_data[..copy_len]);
}
}
@@ -3662,22 +3564,13 @@ fn try_gzip_data(data: &[u8]) -> Option<Vec<u8>> {
encoder.finish().ok()
}
fn maybe_decompress_gzip(data: &[u8]) -> Result<Vec<u8>, GunzipError> {
fn maybe_decompress_gzip(data: &[u8]) -> Option<Vec<u8>> {
use flate2::read::GzDecoder;
use std::io::Read;
// Cap the output so a crafted, highly-compressible (gzip-bomb) needle cannot
// OOM the server when decompressed on read or upload. take(limit+1) lets us
// tell "exactly at the limit" apart from "over it", which we reject as
// TooLarge so callers fail the request instead of silently using raw bytes.
let mut decoder = GzDecoder::new(data).take(MAX_EXPANSION_BYTES + 1);
let mut decoder = GzDecoder::new(data);
let mut decompressed = Vec::new();
decoder
.read_to_end(&mut decompressed)
.map_err(|_| GunzipError::Decode)?;
if decompressed.len() as u64 > MAX_EXPANSION_BYTES {
return Err(GunzipError::TooLarge);
}
Ok(decompressed)
decoder.read_to_end(&mut decompressed).ok()?;
Some(decompressed)
}
fn compute_md5_base64(data: &[u8]) -> String {
@@ -3919,11 +3812,7 @@ mod tests {
let compressed = try_gzip_data(data).unwrap();
let decompressed = maybe_decompress_gzip(&compressed).unwrap();
assert_eq!(decompressed, data);
// Non-gzip input is reported as a decode error (callers fall back to raw).
assert!(matches!(
maybe_decompress_gzip(data),
Err(GunzipError::Decode)
));
assert!(maybe_decompress_gzip(data).is_none());
}
#[test]
+2 -2
View File
@@ -131,8 +131,8 @@ impl DiskLocation {
volume_id = vid.0,
"volume was not completed: {}, removing files", note
);
remove_volume_files(&volume_name, false);
remove_volume_files(&idx_name, false);
remove_volume_files(&volume_name);
remove_volume_files(&idx_name);
continue;
}
+2 -51
View File
@@ -598,16 +598,8 @@ impl Store {
as i32;
let mut max_count = vol_count + ec_equivalent;
// One slot per full volume that fits in the unclaimed space.
// A "- 1" here used to zero the count when the disk had room for
// exactly one volume (free between 1x and 2x the limit), stranding
// auto-sized disks at max_volume_count 0 with no writable volume.
if unclaimed > 0 {
max_count += (unclaimed as u64 / volume_size_limit) as i32;
}
// An auto-sized disk with free space always hosts at least one volume.
if max_count < 1 {
max_count = 1;
if unclaimed > volume_size_limit as i64 {
max_count += (unclaimed as u64 / volume_size_limit) as i32 - 1;
}
loc.max_volume_count.store(max_count, Ordering::Relaxed);
@@ -1470,47 +1462,6 @@ mod tests {
assert!(with_preallocate > without_preallocate);
}
#[test]
fn test_maybe_adjust_volume_max_no_dead_zone() {
// With auto max (original_max_volume_count == 0) and a large volume size
// limit, a disk with room for at least one volume must not report 0
// slots. It once did so for disks sized between 1x and 2x the limit,
// leaving no writable volume and timing out every assign.
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
if free < 4 {
return; // cannot determine free disk space
}
let mut store = Store::new(NeedleMapKind::InMemory);
store
.add_location(
dir,
dir,
0, // auto
DiskType::HardDrive,
MinFreeSpace::Percent(1.0),
Vec::new(),
)
.unwrap();
// free disk is 1.5x the limit: room for one full volume, less than two.
let volume_size_limit = free * 2 / 3;
store
.volume_size_limit
.store(volume_size_limit, Ordering::Relaxed);
store.maybe_adjust_volume_max();
let max = store.locations[0].max_volume_count.load(Ordering::Relaxed);
assert!(
max >= 1,
"auto-sized disk with room for one volume reported max_volume_count={max}; want >= 1"
);
}
#[test]
fn test_find_free_location_predicate_prefers_more_capacity_and_skips_low_disk() {
let tmp1 = TempDir::new().unwrap();
+4 -72
View File
@@ -3071,15 +3071,11 @@ impl Volume {
let bytes = fake_del_needle.write_bytes(version);
dst_dat.write_all(&bytes)?;
// Record the tombstone's real .dat offset, like the normal delete path,
// so a deletion left at the .dat tail stays visible to the integrity
// check on reload. Offset 0 hid the trailing tombstone and falsely
// flipped the volume read-only.
let mut idx_entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
crate::storage::types::idx_entry_to_bytes(
&mut idx_entry_buf,
key,
Offset::from_actual_offset(dat_offset as i64),
Offset::from_actual_offset(0),
Size(crate::storage::types::TOMBSTONE_FILE_SIZE.into()),
);
dst_idx.write_all(&idx_entry_buf)?;
@@ -3161,34 +3157,12 @@ impl Volume {
}
}
// A regular volume and an EC volume for the same id share <base>.vif.
// When EC artefacts coexist on this disk (e.g. shards distributed onto
// a source replica before it is deleted), keep the .vif so removing the
// regular volume does not strip the EC volume's info file.
let keep_vif = self.shares_vif_with_ec_volume();
self.close();
remove_volume_files(&self.data_file_name(), keep_vif);
remove_volume_files(&self.index_file_name(), keep_vif);
remove_volume_files(&self.data_file_name());
remove_volume_files(&self.index_file_name());
Ok(())
}
/// Reports whether an EC volume for this id has a sealed .ecx on the same
/// disk, in which case its .vif is the same file as the regular volume's
/// and must outlive the regular volume's deletion. Mirrors the on-disk
/// portion of Go's Volume.sharesVifWithEcVolume / HasEcxFileOnDisk.
fn shares_vif_with_ec_volume(&self) -> bool {
let has_ecx = |base: &str| -> bool {
fs::metadata(format!("{}.ecx", base))
.map(|m| !m.is_dir() && m.len() > 0)
.unwrap_or(false)
};
if has_ecx(&volume_file_name(&self.dir_idx, &self.collection, self.id)) {
return true;
}
self.dir != self.dir_idx
&& has_ecx(&volume_file_name(&self.dir, &self.collection, self.id))
}
/// Check if an I/O error is EIO (errno 5) and record it for health monitoring.
/// On success (None), clears any previously recorded EIO error.
/// Matches Go's `checkReadWriteError` in volume_write.go.
@@ -3257,13 +3231,10 @@ fn get_append_at_ns(last: u64) -> u64 {
/// Remove all files associated with a volume.
/// .dat/.idx removals log at info level so destructive calls are traceable.
pub(crate) fn remove_volume_files(base: &str, keep_vif: bool) {
pub(crate) fn remove_volume_files(base: &str) {
for ext in &[
".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
] {
if *ext == ".vif" && keep_vif {
continue;
}
let path = format!("{}{}", base, ext);
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
let existed = fs::remove_file(&path).is_ok();
@@ -4769,43 +4740,4 @@ mod tests {
".vif removed from data dir"
);
}
/// When an EC volume for the same id has a sealed .ecx on the same disk, the
/// .vif is shared with it and must survive the regular volume's deletion.
#[test]
fn test_destroy_keeps_vif_when_ec_coexists() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut v = make_test_volume(dir);
let mut n = Needle {
id: NeedleId(1),
cookie: Cookie(1),
data: b"test".to_vec(),
data_size: 4,
..Needle::default()
};
v.write_needle(&mut n, true).unwrap();
let vif_path = format!("{}/1.vif", dir);
std::fs::write(&vif_path, r#"{"version":3}"#).unwrap();
// A sealed .ecx marks a coexisting EC volume for the same id.
let ecx_path = format!("{}/1.ecx", dir);
std::fs::write(&ecx_path, b"ec-index").unwrap();
v.destroy(false, false).unwrap();
let dat_path = format!("{}/1.dat", dir);
let idx_path = format!("{}/1.idx", dir);
assert!(!std::path::Path::new(&dat_path).exists(), ".dat removed");
assert!(!std::path::Path::new(&idx_path).exists(), ".idx removed");
assert!(
std::path::Path::new(&vif_path).exists(),
".vif kept: shared with the coexisting EC volume"
);
assert!(
std::path::Path::new(&ecx_path).exists(),
".ecx is an EC sidecar, never touched here"
);
}
}
BIN
View File
Binary file not shown.
-15
View File
@@ -1,15 +0,0 @@
# Terraform / OpenTofu working files
.terraform/
.terraform.lock.hcl
*.tfstate
*.tfstate.*
crash.log
crash.*.log
*.tfvars
!*.tfvars.example
override.tf
override.tf.json
*_override.tf
*_override.tf.json
.terraformrc
terraform.rc
-128
View File
@@ -1,128 +0,0 @@
# SeaweedFS on Terraform
> **Experimental.** This Terraform support is an early scaffold under active
> development. Interfaces (variables, outputs, module layout) may change without
> notice, and not every tier is implemented yet. Not recommended for production
> without your own review and testing.
Self-contained Terraform/OpenTofu modules to deploy SeaweedFS on cloud VMs
running the `weed` binary directly under systemd. No Helm and no Kubernetes
required.
What works today is verified end-to-end against a real `weed` cluster (see
"Test it locally" below).
## Layout
```text
terraform/
modules/
core/ cloud-agnostic renderer (ZERO cloud resources): turns an
address map + config into per-node weed argv, systemd units,
config files, disk-mount + secret-fetch scripts, and cloud-init.
Both the cloud wrappers and the local harnesses consume `nodes`.
security/ cloud-agnostic CA + per-component mTLS certs (distinct CNs) +
JWT signing keys (tls/random providers). Emits a `core_security`
object ready for the core, plus PEMs for secret-store delivery.
aws/ thin AWS wrapper: reserves stable ENIs (fixed private IPs)
first, feeds them to core, creates instances + protected EBS
data disks + SG; with enable_security it generates certs/JWT,
stores them in SSM SecureString, grants an instance role, and
renders a boot fetch-secrets.sh. Keyed for_each throughout.
examples/
aws-ha-distributed/ 3 masters + 3 volumes (1/AZ) + 2 filers + 1 S3,
secure-by-default (mTLS via SSM-delivered certs)
aws-all-in-one/ single `weed server` instance via core directly
test/
local/ render a cluster with core and run it as real weed processes
on 127.0.0.1 (no cloud, no docker), then assert it works
local-secure/ generate certs/JWT with the security module, run a real
mTLS cluster, and assert it forms + enforces JWT auth
```
## Design in one paragraph
The chart is the structural reference, not a dependency. A cloud-agnostic
**core** renders everything portable; thin per-cloud wrappers provision infra.
Addressing is an **input** to the core (wrapper reserves static IPs first), so
the wrapper -> core dependency is one-way with no apply-time cycle. Stateful
tiers (master/volume/filer) are keyed `for_each` maps, never `count`, so a
middle node can be replaced without reindexing its peers or reattaching the
wrong disk. Flag names are verified against the real `weed` binary
(notably: volume uses `-mserver` for the master list; gRPC is `-port.grpc`,
auto = http+10000; `minFreeSpacePercent` is a string).
## Test it locally
Requires `tofu` (or `terraform`), `jq`, `curl`, and a `weed` binary. Renders a
3-master + volume + filer + S3 cluster from the core module and runs it as real
processes, asserting quorum, volume registration, and filer/S3 round-trips:
```bash
cd terraform/test/local
WEED=/path/to/weed ./run_local_cluster.sh
# => 7 passed, 0 failed
```
The harness uses a high port range (29333/28080/28888/28333) so it does not
collide with a SeaweedFS cluster already running on the machine, and aborts if a
required port is taken. `KEEP=1 ./run_local_cluster.sh` leaves the cluster up.
### mTLS end-to-end
```bash
cd terraform/test/local-secure
WEED=/path/to/weed ./run_local_secure.sh
# => generates a CA + component certs + JWT, renders security.toml, runs a real
# mTLS cluster, asserts master/volume/filer form over mutual TLS and that the
# filer enforces JWT signing (unsigned writes get 401). 5 passed.
```
## Plan-level tests (no cloud)
```bash
cd terraform/modules/core && tofu test
# => 11 passed: peers list, -mserver vs -master, metrics gating,
# security.toml conditions, all-in-one inheritance, ...
```
## Validate the cloud wrappers
```bash
cd terraform/examples/aws-ha-distributed && tofu init && tofu validate
```
`apply` needs AWS credentials, a VPC, subnets, and an AMI with `weed` installed
(bake with Packer, or install at boot).
## Status
Implemented and verified:
- Tiers: master / volume / filer / s3 / all-in-one rendered by the core.
- Disk mount: cloud-init runs a `mount-disks.sh` that auto-discovers (or takes
explicit candidate devices), `blkid`-guards `mkfs`, mounts, and persists to
`/etc/fstab` by UUID. The AWS wrapper wires the protected EBS disk to `/data`.
- mTLS + JWT: the `security` module generates the CA + per-component certs
(distinct CNs) + JWT keys; the AWS wrapper stores them in SSM SecureString,
grants an instance role, and the core renders a boot fetch script so secrets
stay OUT of user_data. Proven end-to-end by `test/local-secure`.
Not yet implemented:
- sftp / admin / worker tiers (core does not render them yet).
- GCP/Azure wrappers, the data-plane provider, and the K8s-native module.
- CA in Vault PKI (currently TF-generated, so the CA key lives in state). KMS
decrypt in the IAM policy is scoped by
`ViaService` but uses `Resource="*"`; tighten to the SSM CMK in production.
## Gotchas
- This `weed` build starts an Iceberg REST catalog on `:8181` by default; set
`s3.iceberg_port = 0` to disable, or a port to relocate it.
- Disk auto-discovery assumes a single attached data disk per node. For multiple
data disks, pass explicit `devices` candidates in `disk_mounts`.
## Security note
`tofu output`/state can contain secrets. Generate secrets outside Terraform
(cloud secret manager / Vault) and fetch them at boot; never commit secret
material.
-133
View File
@@ -1,133 +0,0 @@
# SeaweedFS all-in-one on a single AWS instance.
#
# Demonstrates the layered design directly: call the cloud-agnostic core to
# render cloud-init for a `weed server` node, then attach it to one instance
# with a protected data disk. No wrapper module needed for the simple case.
#
# tofu init && tofu validate
# tofu apply # requires AWS credentials + a weed AMI
terraform {
required_version = ">= 1.3.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.40"
}
}
}
provider "aws" {
region = var.region
}
variable "region" {
type = string
default = "us-east-1"
}
variable "vpc_id" {
type = string
}
variable "subnet_id" {
type = string
}
variable "availability_zone" {
type = string
default = "us-east-1a"
}
variable "ami_id" {
type = string
}
variable "private_ip" {
type = string
default = "10.0.1.50"
}
module "core" {
source = "../../modules/core"
weed_binary = "/usr/bin/weed"
# disable the distributed tiers; run a single all-in-one process
master = { enabled = false }
volume = { enabled = false }
filer = { enabled = false }
all_in_one = {
enabled = true
nodes = { a0 = { address = var.private_ip, data_dir = "/data" } }
s3 = { enabled = true }
}
s3_identities = [{
name = "anonymous"
access_key = ""
secret_key = ""
actions = ["Read", "List"]
}]
}
resource "aws_network_interface" "aio" {
subnet_id = var.subnet_id
private_ips = [var.private_ip]
security_groups = [aws_security_group.aio.id]
tags = { Name = "seaweedfs-all-in-one" }
}
resource "aws_security_group" "aio" {
name_prefix = "seaweedfs-aio-"
vpc_id = var.vpc_id
lifecycle {
create_before_destroy = true
}
}
resource "aws_vpc_security_group_ingress_rule" "s3" {
security_group_id = aws_security_group.aio.id
cidr_ipv4 = "10.0.0.0/8"
ip_protocol = "tcp"
from_port = 8333
to_port = 8333
}
resource "aws_vpc_security_group_egress_rule" "all" {
security_group_id = aws_security_group.aio.id
cidr_ipv4 = "0.0.0.0/0"
ip_protocol = "-1"
}
resource "aws_instance" "aio" {
ami = var.ami_id
instance_type = "m5.large"
user_data = module.core.cloud_init_by_node["all-in-one-a0"]
network_interface {
network_interface_id = aws_network_interface.aio.id
device_index = 0
}
metadata_options {
http_tokens = "required"
}
tags = { Name = "seaweedfs-all-in-one", Role = "all-in-one" }
}
resource "aws_ebs_volume" "data" {
availability_zone = var.availability_zone
size = 200
type = "gp3"
encrypted = true
tags = { Name = "seaweedfs-all-in-one-data" }
lifecycle {
prevent_destroy = true
}
}
resource "aws_volume_attachment" "data" {
device_name = "/dev/xvdf"
volume_id = aws_ebs_volume.data.id
instance_id = aws_instance.aio.id
stop_instance_before_detaching = true
}
output "instance_id" {
value = aws_instance.aio.id
}
@@ -1,117 +0,0 @@
# SeaweedFS HA on AWS: 3-master quorum + 3 volume servers (one per AZ) + 2
# filers (leveldb2-replicated HA) + 1 standalone S3 gateway.
#
# tofu init && tofu validate
# tofu apply # requires AWS credentials, a VPC, subnets, and a weed AMI
#
# This is a scaffold: it provisions instances, protected EBS data disks, and the
# security group. Mounting the EBS disk at /data and secret-store cert delivery
# are documented follow-ups (see terraform/README.md).
terraform {
required_version = ">= 1.3.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.40"
}
}
}
provider "aws" {
region = var.region
}
variable "region" {
type = string
default = "us-east-1"
}
variable "vpc_id" {
type = string
}
variable "ami_id" {
description = "AMI with the weed binary at /usr/bin/weed."
type = string
}
# subnet per AZ
variable "subnet_a" { type = string }
variable "subnet_b" { type = string }
variable "subnet_c" { type = string }
# AZ defaults assume region us-east-1. When you change `region`, override az_a/az_b/az_c
# (and the matching subnets) with AZs that belong to that region, or the volume/filer
# EBS volumes will fail to create in a mismatched AZ.
variable "az_a" {
type = string
default = "us-east-1a"
}
variable "az_b" {
type = string
default = "us-east-1b"
}
variable "az_c" {
type = string
default = "us-east-1c"
}
variable "client_ingress_cidrs" {
type = list(string)
default = ["10.0.0.0/8"]
}
variable "ssh_ingress_cidrs" {
type = list(string)
default = []
}
module "seaweedfs" {
source = "../../modules/aws"
name = "seaweedfs"
vpc_id = var.vpc_id
ami_id = var.ami_id
# secure-by-default flagship: mTLS (certs + JWT generated by the security
# submodule, delivered via SSM and fetched at boot) plus monitoring.
enable_security = true
monitoring_enabled = true
masters = {
m0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.10" }
m1 = { subnet_id = var.subnet_b, private_ip = "10.0.2.10" }
m2 = { subnet_id = var.subnet_c, private_ip = "10.0.3.10" }
}
volumes = {
v0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.20", rack = var.az_a, data_center = var.region, data_volume_size_gb = 500 }
v1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.20", rack = var.az_b, data_center = var.region, data_volume_size_gb = 500 }
v2 = { subnet_id = var.subnet_c, availability_zone = var.az_c, private_ip = "10.0.3.20", rack = var.az_c, data_center = var.region, data_volume_size_gb = 500 }
}
filers = {
f0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.30" }
f1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.30" }
}
s3_nodes = {
s0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.40" }
}
client_ingress_cidrs = var.client_ingress_cidrs
ssh_ingress_cidrs = var.ssh_ingress_cidrs
}
output "master_peers" {
value = module.seaweedfs.master_peers
}
output "instance_ids" {
value = module.seaweedfs.instance_ids
}
output "security_group_id" {
value = module.seaweedfs.security_group_id
}
-205
View File
@@ -1,205 +0,0 @@
# ---- step 1: reserve stable addressing (ENIs with fixed private IPs) -------
# These exist BEFORE the core renders config, so the address map is known and
# the core <- wrapper dependency is one-way (no apply-time cycle).
resource "aws_network_interface" "master" {
for_each = var.masters
subnet_id = each.value.subnet_id
private_ips = [each.value.private_ip]
security_groups = [aws_security_group.cluster.id]
tags = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" })
}
resource "aws_network_interface" "volume" {
for_each = var.volumes
subnet_id = each.value.subnet_id
private_ips = [each.value.private_ip]
security_groups = [aws_security_group.cluster.id]
tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" })
}
resource "aws_network_interface" "filer" {
for_each = var.filers
subnet_id = each.value.subnet_id
private_ips = [each.value.private_ip]
security_groups = [aws_security_group.cluster.id]
tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" })
}
resource "aws_network_interface" "s3" {
for_each = var.s3_nodes
subnet_id = each.value.subnet_id
private_ips = [each.value.private_ip]
security_groups = [aws_security_group.cluster.id]
tags = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" })
}
# ---- step 2: render config with the cloud-agnostic core --------------------
module "core" {
source = "../core"
weed_binary = var.weed_binary
monitoring_enabled = var.monitoring_enabled
enable_security = var.enable_security
security = local.core_security
# Keep secrets out of cloud-init user_data when security is on; deliver them
# from SSM via the boot fetch script instead.
render_secret_files = !var.enable_security
boot_fetch_script = local.fetch_script
master = {
nodes = { for k, v in var.masters : k => { address = v.private_ip } }
}
volume = {
enabled = length(var.volumes) > 0
nodes = { for k, v in var.volumes : k => {
address = v.private_ip
rack = v.rack
data_center = v.data_center
data_dirs = [{ path = "/data", max_volumes = 0 }]
# auto-discover the single attached data disk and mount it at /data
disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }]
} }
}
filer = {
enabled = length(var.filers) > 0
nodes = { for k, v in var.filers : k => {
address = v.private_ip
data_dir = "/data/filerldb2"
disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }]
} }
s3 = var.embedded_s3
}
s3 = {
enabled = length(var.s3_nodes) > 0
nodes = { for k, v in var.s3_nodes : k => { address = v.private_ip } }
}
s3_identities = var.s3_identities
}
# ---- step 3: instances (keyed for_each; user_data = rendered cloud-init) ----
resource "aws_instance" "master" {
for_each = var.masters
ami = var.ami_id
instance_type = each.value.instance_type
key_name = var.key_name
user_data = module.core.cloud_init_by_node["master-${each.key}"]
iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null
disable_api_termination = var.termination_protection
network_interface {
network_interface_id = aws_network_interface.master[each.key].id
device_index = 0
}
metadata_options {
http_tokens = "required" # IMDSv2
}
tags = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" })
}
resource "aws_instance" "volume" {
for_each = var.volumes
ami = var.ami_id
instance_type = each.value.instance_type
key_name = var.key_name
user_data = module.core.cloud_init_by_node["volume-${each.key}"]
iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null
disable_api_termination = var.termination_protection
network_interface {
network_interface_id = aws_network_interface.volume[each.key].id
device_index = 0
}
metadata_options {
http_tokens = "required"
}
tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" })
}
resource "aws_instance" "filer" {
for_each = var.filers
ami = var.ami_id
instance_type = each.value.instance_type
key_name = var.key_name
user_data = module.core.cloud_init_by_node["filer-${each.key}"]
iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null
disable_api_termination = var.termination_protection
network_interface {
network_interface_id = aws_network_interface.filer[each.key].id
device_index = 0
}
metadata_options {
http_tokens = "required"
}
tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" })
}
resource "aws_instance" "s3" {
for_each = var.s3_nodes
ami = var.ami_id
instance_type = each.value.instance_type
key_name = var.key_name
user_data = module.core.cloud_init_by_node["s3-${each.key}"]
iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null
network_interface {
network_interface_id = aws_network_interface.s3[each.key].id
device_index = 0
}
metadata_options {
http_tokens = "required"
}
tags = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" })
}
# ---- step 4: protected data disks (decoupled; survive instance replacement) -
# The core's mount-disks.sh (wired via disk_mounts above) mkfs+mounts these at
# /data before the weed unit starts.
resource "aws_ebs_volume" "volume_data" {
for_each = var.volumes
availability_zone = each.value.availability_zone
size = each.value.data_volume_size_gb
type = each.value.data_volume_type
iops = each.value.data_volume_iops
encrypted = true
tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}-data", Role = "volume" })
lifecycle {
prevent_destroy = true # break-glass: remove to allow destroy
}
}
resource "aws_volume_attachment" "volume_data" {
for_each = var.volumes
device_name = "/dev/xvdf"
volume_id = aws_ebs_volume.volume_data[each.key].id
instance_id = aws_instance.volume[each.key].id
stop_instance_before_detaching = true
}
resource "aws_ebs_volume" "filer_data" {
for_each = var.filers
availability_zone = each.value.availability_zone
size = each.value.data_volume_size_gb
type = "gp3"
encrypted = true
tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}-data", Role = "filer" })
lifecycle {
prevent_destroy = true
}
}
resource "aws_volume_attachment" "filer_data" {
for_each = var.filers
device_name = "/dev/xvdf"
volume_id = aws_ebs_volume.filer_data[each.key].id
instance_id = aws_instance.filer[each.key].id
stop_instance_before_detaching = true
}
-34
View File
@@ -1,34 +0,0 @@
output "security_group_id" {
description = "Cluster security group id."
value = aws_security_group.cluster.id
}
output "master_private_ips" {
description = "Master private IPs keyed by node id."
value = { for k, v in var.masters : k => v.private_ip }
}
output "master_peers" {
description = "Master peer list passed to the weed processes."
value = module.core.master_peers
}
output "filer_private_ips" {
description = "Filer private IPs keyed by node id."
value = { for k, v in var.filers : k => v.private_ip }
}
output "s3_private_ips" {
description = "Standalone S3 gateway private IPs keyed by node id."
value = { for k, v in var.s3_nodes : k => v.private_ip }
}
output "instance_ids" {
description = "All instance ids keyed by role-node."
value = merge(
{ for k, v in aws_instance.master : "master-${k}" => v.id },
{ for k, v in aws_instance.volume : "volume-${k}" => v.id },
{ for k, v in aws_instance.filer : "filer-${k}" => v.id },
{ for k, v in aws_instance.s3 : "s3-${k}" => v.id },
)
}
-136
View File
@@ -1,136 +0,0 @@
# =============================================================================
# mTLS + JWT generation and secret delivery (active when enable_security=true).
#
# Generates the CA + per-component certs + JWT keys with the security submodule,
# stores them (plus the rendered security.toml / S3 config) in SSM Parameter
# Store as SecureString, and renders a boot fetch-secrets.sh that pulls them via
# the instance role. This keeps secrets OUT of cloud-init user_data / IMDS.
# Secrets still live in Terraform state (TF-generated); prefer Vault PKI for the
# CA in production.
# =============================================================================
locals {
all_private_ips = concat(
[for k, v in var.masters : v.private_ip],
[for k, v in var.volumes : v.private_ip],
[for k, v in var.filers : v.private_ip],
[for k, v in var.s3_nodes : v.private_ip],
)
ssm_prefix = "/seaweedfs/${var.name}"
deliver_s3_config = length(var.s3_nodes) > 0 || var.embedded_s3.enabled
# security.toml is identical across nodes; take it from the first master.
first_master = var.enable_security ? sort(keys(var.masters))[0] : ""
security_toml_path = "/etc/seaweedfs/security.toml"
s3_config_path = "/etc/seaweedfs/s3_config.json"
# Boot fetch entries: cert files + security.toml + (optional) S3 config.
fetch_entries = var.enable_security ? concat(
[for m in module.security[0].secret_manifest : { param = "${local.ssm_prefix}/certs/${m.key}", path = m.path, mode = m.mode }],
[{ param = "${local.ssm_prefix}/security.toml", path = local.security_toml_path, mode = "0600" }],
local.deliver_s3_config ? [{ param = "${local.ssm_prefix}/s3_config.json", path = local.s3_config_path, mode = "0600" }] : [],
) : []
fetch_script = var.enable_security ? templatefile("${path.module}/templates/fetch-secrets.sh.tftpl", {
run_as_user = "seaweedfs"
entries = local.fetch_entries
}) : ""
core_security = var.enable_security ? module.security[0].core_security : var.security
}
module "security" {
count = var.enable_security ? 1 : 0
source = "../security"
internal_domain = var.internal_domain
cert_dir = var.cert_dir
ip_sans = local.all_private_ips
}
# ---- SSM SecureString parameters -------------------------------------------
resource "aws_ssm_parameter" "cert" {
for_each = var.enable_security ? { for m in module.security[0].secret_manifest : m.key => m } : {}
name = "${local.ssm_prefix}/certs/${each.key}"
type = "SecureString"
value = module.security[0].secret_contents[each.key]
tags = var.tags
}
resource "aws_ssm_parameter" "security_toml" {
count = var.enable_security ? 1 : 0
name = "${local.ssm_prefix}/security.toml"
type = "SecureString"
value = module.core.secret_files_by_node["master-${local.first_master}"][local.security_toml_path]
tags = var.tags
}
resource "aws_ssm_parameter" "s3_config" {
count = var.enable_security && local.deliver_s3_config ? 1 : 0
name = "${local.ssm_prefix}/s3_config.json"
type = "SecureString"
# identical content across s3/filer nodes; render once via the security module's
# JSON is not available here, so read it back from any node that carries it.
value = local.s3_config_source
tags = var.tags
}
locals {
# Pull the rendered S3 identity JSON from whichever node carries it.
s3_config_source = !local.deliver_s3_config ? "" : (
length(var.s3_nodes) > 0 ?
module.core.secret_files_by_node["s3-${sort(keys(var.s3_nodes))[0]}"][local.s3_config_path] :
(length(var.filers) > 0 ? module.core.secret_files_by_node["filer-${sort(keys(var.filers))[0]}"][local.s3_config_path] : "")
)
}
# ---- IAM: instance role allowed to read the SSM params ----------------------
data "aws_iam_policy_document" "assume" {
count = var.enable_security ? 1 : 0
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
}
}
data "aws_iam_policy_document" "ssm_read" {
count = var.enable_security ? 1 : 0
statement {
sid = "ReadSeaweedfsParams"
actions = ["ssm:GetParameter", "ssm:GetParameters"]
resources = ["arn:aws:ssm:*:*:parameter${local.ssm_prefix}/*"]
}
statement {
sid = "DecryptSecureStrings"
actions = ["kms:Decrypt"]
resources = [var.kms_key_arn] # defaults to "*"; set a CMK ARN in production
condition {
test = "StringEquals"
variable = "kms:ViaService"
values = ["ssm.*.amazonaws.com"]
}
}
}
resource "aws_iam_role" "node" {
count = var.enable_security ? 1 : 0
name_prefix = "${var.name}-node-"
assume_role_policy = data.aws_iam_policy_document.assume[0].json
tags = var.tags
}
resource "aws_iam_role_policy" "ssm_read" {
count = var.enable_security ? 1 : 0
name = "ssm-read"
role = aws_iam_role.node[0].id
policy = data.aws_iam_policy_document.ssm_read[0].json
}
resource "aws_iam_instance_profile" "node" {
count = var.enable_security ? 1 : 0
name_prefix = "${var.name}-node-"
role = aws_iam_role.node[0].name
}
-62
View File
@@ -1,62 +0,0 @@
# Cluster security group + the SeaweedFS port matrix.
# Intra-cluster: all TCP allowed within the SG (master 9333/19333, volume
# 8080/18080, filer 8888/18888, admin 33646, ...). Client-facing: only S3 (8333)
# and filer (8888). Metrics ports (9327) are never opened to clients.
resource "aws_security_group" "cluster" {
name_prefix = "${var.name}-cluster-"
description = "SeaweedFS cluster"
vpc_id = var.vpc_id
tags = merge(var.tags, { Name = "${var.name}-cluster" })
lifecycle {
create_before_destroy = true
}
}
# All intra-cluster TCP via SG self-reference.
resource "aws_vpc_security_group_ingress_rule" "intra_cluster" {
security_group_id = aws_security_group.cluster.id
referenced_security_group_id = aws_security_group.cluster.id
ip_protocol = "tcp"
from_port = 0
to_port = 65535
description = "intra-cluster (http + gRPC for all roles)"
}
resource "aws_vpc_security_group_ingress_rule" "s3" {
for_each = toset(var.client_ingress_cidrs)
security_group_id = aws_security_group.cluster.id
cidr_ipv4 = each.value
ip_protocol = "tcp"
from_port = 8333
to_port = 8333
description = "S3 gateway"
}
resource "aws_vpc_security_group_ingress_rule" "filer" {
for_each = toset(var.client_ingress_cidrs)
security_group_id = aws_security_group.cluster.id
cidr_ipv4 = each.value
ip_protocol = "tcp"
from_port = 8888
to_port = 8888
description = "filer HTTP"
}
resource "aws_vpc_security_group_ingress_rule" "ssh" {
for_each = toset(var.ssh_ingress_cidrs)
security_group_id = aws_security_group.cluster.id
cidr_ipv4 = each.value
ip_protocol = "tcp"
from_port = 22
to_port = 22
description = "SSH"
}
resource "aws_vpc_security_group_egress_rule" "all" {
security_group_id = aws_security_group.cluster.id
cidr_ipv4 = "0.0.0.0/0"
ip_protocol = "-1"
description = "all egress"
}
@@ -1,21 +0,0 @@
#!/usr/bin/env bash
# Rendered by terraform-aws-seaweedfs. Pulls certs + secret config from SSM
# Parameter Store (SecureString) at boot using the instance role. Region is
# auto-resolved by the AWS CLI from IMDS, so no region is baked in. Requires
# the AWS CLI and a working instance profile (see security.tf).
set -euo pipefail
run_as_user="${run_as_user}"
fetch() {
local param="$1" dest="$2" mode="$3"
mkdir -p "$(dirname "$dest")"
aws ssm get-parameter --with-decryption --name "$param" \
--query 'Parameter.Value' --output text > "$dest"
chmod "$mode" "$dest"
chown "$run_as_user:$run_as_user" "$dest" 2>/dev/null || true
}
%{ for e in entries ~}
fetch "${e.param}" "${e.path}" "${e.mode}"
%{ endfor ~}
-171
View File
@@ -1,171 +0,0 @@
# =============================================================================
# terraform-aws-seaweedfs - thin AWS wrapper around the cloud-agnostic core.
#
# Reserves stable per-node addressing (ENIs with fixed private IPs) FIRST, feeds
# that address map to the core to render cloud-init, then creates instances and
# protected EBS data disks. All tiers are keyed for_each (never count), so a
# middle node can be replaced without reindexing its peers/disks.
# =============================================================================
variable "name" {
description = "Name prefix for all resources."
type = string
default = "seaweedfs"
}
variable "vpc_id" {
description = "VPC to deploy into."
type = string
}
variable "ami_id" {
description = "AMI with the weed binary installed (e.g. baked with Packer)."
type = string
}
variable "weed_binary" {
description = "Path to weed on the AMI."
type = string
default = "/usr/bin/weed"
}
variable "key_name" {
description = "EC2 key pair for SSH (optional)."
type = string
default = null
}
variable "monitoring_enabled" {
description = "Bind metrics ports and pass them through to the core."
type = bool
default = false
}
variable "enable_security" {
description = "Enable mTLS (security.toml). Cert material must be supplied via the security variable / secret store."
type = bool
default = false
}
variable "security" {
description = "Passed through to the core security variable when enable_security=false. When enable_security=true this is ignored and certs/JWT are generated by the security submodule."
type = any
default = {}
}
variable "internal_domain" {
description = "Internal domain for generated component cert CNs / peer-auth wildcard (enable_security=true)."
type = string
default = "seaweedfs.internal"
}
variable "cert_dir" {
description = "On-host directory where certs are fetched/placed."
type = string
default = "/usr/local/share/ca-certificates"
}
variable "kms_key_arn" {
description = "KMS key ARN the instance role may use to decrypt SSM SecureStrings. Defaults to \"*\" (the SSM default key); set to a specific CMK ARN in production."
type = string
default = "*"
}
variable "termination_protection" {
description = "Set disable_api_termination on stateful instances (masters, volumes)."
type = bool
default = true
}
# ---- stateful tiers: keyed node maps ---------------------------------------
variable "masters" {
description = "Master nodes keyed by stable id (m0/m1/m2). Quorum size must be odd."
type = map(object({
subnet_id = string
private_ip = string
instance_type = optional(string, "t3.medium")
}))
validation {
condition = length(var.masters) % 2 == 1
error_message = "Master quorum must be an odd number of nodes (1, 3, 5)."
}
}
variable "volumes" {
description = "Volume nodes keyed by stable id. Each owns one protected EBS data disk."
type = map(object({
subnet_id = string
availability_zone = string
private_ip = string
instance_type = optional(string, "m5.large")
rack = optional(string)
data_center = optional(string)
data_volume_size_gb = optional(number, 100)
data_volume_type = optional(string, "gp3")
data_volume_iops = optional(number, null)
}))
default = {}
}
variable "filers" {
description = "Filer nodes keyed by stable id. Each owns a protected EBS disk for its leveldb2 store."
type = map(object({
subnet_id = string
availability_zone = string
private_ip = string
instance_type = optional(string, "t3.medium")
data_volume_size_gb = optional(number, 50)
}))
default = {}
}
variable "s3_nodes" {
description = "Standalone S3 gateway nodes keyed by stable id (stateless)."
type = map(object({
subnet_id = string
private_ip = string
instance_type = optional(string, "t3.medium")
}))
default = {}
}
variable "embedded_s3" {
description = "Embed the S3 gateway in the filer instead of standalone nodes."
type = object({
enabled = optional(bool, false)
port = optional(number, 8333)
domain_name = optional(string, "")
})
default = {}
}
variable "s3_identities" {
description = "Non-admin S3 identities for the gateway config (admin key goes via EnvironmentFile)."
type = list(object({
name = string
access_key = string
secret_key = string
actions = list(string)
}))
default = []
sensitive = true
}
# ---- networking ------------------------------------------------------------
variable "client_ingress_cidrs" {
description = "CIDRs allowed to reach the client-facing S3 (8333) and filer (8888) ports."
type = list(string)
default = []
}
variable "ssh_ingress_cidrs" {
description = "CIDRs allowed SSH (22)."
type = list(string)
default = []
}
variable "tags" {
description = "Extra tags applied to all resources."
type = map(string)
default = {}
}
-12
View File
@@ -1,12 +0,0 @@
terraform {
required_version = ">= 1.3.0"
required_providers {
aws = {
source = "hashicorp/aws"
# >= 5.40 for the modern aws_vpc_security_group_*_rule resources.
# Provider >= 6.0 is recommended for the EBS-attachment no-replace behavior;
# bump the floor here once you standardize on it.
version = ">= 5.40"
}
}
}
-373
View File
@@ -1,373 +0,0 @@
# =============================================================================
# SeaweedFS Terraform core - pure renderer (zero cloud resources).
#
# Computes, per node: the verified `weed` argv, the full systemd ExecStart,
# config files, environment, and rendered systemd unit + cloud-init.
# =============================================================================
locals {
# ---- shared security.toml (rendered when mTLS on OR any JWT key set) ----
jwt_any = anytrue([
var.security.jwt_signing_key != "",
var.security.jwt_signing_read_key != "",
var.security.jwt_filer_signing_key != "",
var.security.jwt_filer_signing_read_key != "",
])
render_security = var.enable_security || local.jwt_any
security_toml = local.render_security ? templatefile("${path.module}/templates/security.toml.tftpl", {
enable_security = var.enable_security
cert_dir = var.security.cert_dir
allowed_wildcard_domain = var.security.allowed_wildcard_domain
allowed_common_names = var.security.allowed_common_names
jwt_signing_key = var.security.jwt_signing_key
jwt_signing_read_key = var.security.jwt_signing_read_key
jwt_filer_signing_key = var.security.jwt_filer_signing_key
jwt_filer_signing_read_key = var.security.jwt_filer_signing_read_key
}) : ""
security_files = local.render_security ? { "/etc/seaweedfs/security.toml" = local.security_toml } : {}
# ---- S3 identity JSON (non-admin identities only) ----
# Empty access_key => anonymous identity (no credentials block), per SeaweedFS
# convention where the identity literally named "anonymous" grants anon access.
s3_config_json = jsonencode({
identities = [for id in var.s3_identities : {
name = id.name
credentials = id.access_key != "" ? [{ accessKey = id.access_key, secretKey = id.secret_key }] : []
actions = id.actions
}]
})
# ---- global flags (before the subcommand) ----
global_flags = compact([
format("-v=%d", var.log_level),
var.log_to_stderr ? "-logtostderr=true" : "",
])
# ---- master peers (sorted by key for determinism) ----
master_keys = sort(keys(var.master.nodes))
master_peer_list = [for k in local.master_keys :
format("%s:%d", var.master.nodes[k].address, coalesce(var.master.nodes[k].port, var.master.port))
]
master_peers = join(",", local.master_peer_list)
# ---- first filer (default s3 target + cluster env) ----
filer_keys = sort(keys(var.filer.nodes))
first_filer = length(local.filer_keys) > 0 ? format("%s:%d", var.filer.nodes[local.filer_keys[0]].address, coalesce(var.filer.nodes[local.filer_keys[0]].port, var.filer.port)) : ""
# ---- cluster discovery env (WEED_CLUSTER_*) ----
cluster_env = merge(
{ WEED_CLUSTER_DEFAULT = var.cluster_name },
local.master_peers != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_MASTER" = local.master_peers } : {},
local.first_filer != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_FILER" = local.first_filer } : {},
)
metrics_flags_master = var.monitoring_enabled ? compact([
format("-metricsPort=%d", var.master.metrics_port),
var.master.metrics_ip != "" ? format("-metricsIp=%s", var.master.metrics_ip) : "",
]) : []
metrics_flags_volume = var.monitoring_enabled ? compact([
format("-metricsPort=%d", var.volume.metrics_port),
var.volume.metrics_ip != "" ? format("-metricsIp=%s", var.volume.metrics_ip) : "",
]) : []
metrics_flags_filer = var.monitoring_enabled ? compact([
format("-metricsPort=%d", var.filer.metrics_port),
var.filer.metrics_ip != "" ? format("-metricsIp=%s", var.filer.metrics_ip) : "",
]) : []
metrics_flags_s3 = var.monitoring_enabled ? [format("-metricsPort=%d", var.s3.metrics_port)] : []
# ===========================================================================
# MASTER nodes
# ===========================================================================
master_nodes = var.master.enabled ? {
for k in local.master_keys : "master-${k}" => {
role = "master"
name = "master-${k}"
address = var.master.nodes[k].address
disk_mounts = var.master.nodes[k].disk_mounts
data_dir = coalesce(var.master.nodes[k].data_dir, var.master.data_dir)
data_dirs = [coalesce(var.master.nodes[k].data_dir, var.master.data_dir)]
ports = {
http = coalesce(var.master.nodes[k].port, var.master.port)
metrics = var.master.metrics_port
}
env = local.cluster_env
config_files = local.security_files
argv = concat(
local.global_flags,
["master"],
[
format("-port=%d", coalesce(var.master.nodes[k].port, var.master.port)),
format("-mdir=%s", coalesce(var.master.nodes[k].data_dir, var.master.data_dir)),
format("-ip=%s", var.master.nodes[k].address),
format("-ip.bind=%s", var.master.ip_bind),
format("-peers=%s", local.master_peers),
format("-defaultReplication=%s", var.master.default_replication),
format("-volumeSizeLimitMB=%d", var.master.volume_size_limit_mb),
format("-electionTimeout=%s", var.master.election_timeout),
format("-heartbeatInterval=%s", var.master.heartbeat_interval),
],
var.master.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.master.nodes[k].grpc_port)] : (
var.master.grpc_port != null ? [format("-port.grpc=%d", var.master.grpc_port)] : []),
var.master.volume_preallocate ? ["-volumePreallocate"] : [],
var.master.garbage_threshold != null ? [format("-garbageThreshold=%s", tostring(var.master.garbage_threshold))] : [],
var.master.raft_hashicorp ? ["-raftHashicorp"] : [],
var.master.resume_state ? ["-resumeState"] : [],
var.master.disable_http ? ["-disableHttp"] : [],
var.master.white_list != "" ? [format("-whiteList=%s", var.master.white_list)] : [],
local.metrics_flags_master,
var.master.extra_args,
)
}
} : {}
# ===========================================================================
# VOLUME nodes
# ===========================================================================
volume_keys = sort(keys(var.volume.nodes))
volume_nodes = var.volume.enabled ? {
for k in local.volume_keys : "volume-${k}" => {
role = "volume"
name = "volume-${k}"
address = var.volume.nodes[k].address
disk_mounts = var.volume.nodes[k].disk_mounts
data_dirs = [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path]
data_dir = (var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs)[0].path
ports = {
http = coalesce(var.volume.nodes[k].port, var.volume.port)
metrics = var.volume.metrics_port
}
env = local.cluster_env
config_files = local.security_files
argv = concat(
local.global_flags,
["volume"],
[
format("-port=%d", coalesce(var.volume.nodes[k].port, var.volume.port)),
format("-dir=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path])),
format("-max=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : tostring(d.max_volumes)])),
format("-ip=%s", var.volume.nodes[k].address),
format("-ip.bind=%s", var.volume.ip_bind),
format("-mserver=%s", local.master_peers),
format("-readMode=%s", var.volume.read_mode),
format("-compactionMBps=%d", var.volume.compaction_mbps),
format("-minFreeSpacePercent=%s", var.volume.min_free_space_percent),
format("-index=%s", var.volume.index),
],
var.volume.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.volume.nodes[k].grpc_port)] : [],
(var.volume.nodes[k].idx_dir != null && var.volume.nodes[k].idx_dir != "") ? [format("-dir.idx=%s", var.volume.nodes[k].idx_dir)] : (var.volume.idx_dir != "" ? [format("-dir.idx=%s", var.volume.idx_dir)] : []),
var.volume.nodes[k].data_center != null ? [format("-dataCenter=%s", var.volume.nodes[k].data_center)] : [],
var.volume.nodes[k].rack != null ? [format("-rack=%s", var.volume.nodes[k].rack)] : [],
var.volume.nodes[k].public_url != null ? [format("-publicUrl=%s", var.volume.nodes[k].public_url)] : [],
var.volume.file_size_limit_mb != null ? [format("-fileSizeLimitMB=%d", var.volume.file_size_limit_mb)] : [],
var.volume.images_fix_orientation ? ["-images.fix.orientation"] : [],
var.volume.white_list != "" ? [format("-whiteList=%s", var.volume.white_list)] : [],
local.metrics_flags_volume,
var.volume.extra_args,
)
}
} : {}
# ===========================================================================
# FILER nodes (+ optional embedded S3)
# ===========================================================================
filer_s3_files = var.filer.s3.enabled ? { (var.filer.s3.config_path) = local.s3_config_json } : {}
filer_nodes = var.filer.enabled ? {
for k in local.filer_keys : "filer-${k}" => {
role = "filer"
name = "filer-${k}"
address = var.filer.nodes[k].address
disk_mounts = var.filer.nodes[k].disk_mounts
data_dir = coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)
data_dirs = [coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)]
ports = {
http = coalesce(var.filer.nodes[k].port, var.filer.port)
metrics = var.filer.metrics_port
}
env = merge(local.cluster_env, var.filer.extra_env)
config_files = merge(local.security_files, local.filer_s3_files)
argv = concat(
local.global_flags,
["filer"],
[
format("-port=%d", coalesce(var.filer.nodes[k].port, var.filer.port)),
format("-ip=%s", var.filer.nodes[k].address),
format("-ip.bind=%s", var.filer.ip_bind),
format("-master=%s", local.master_peers),
format("-defaultReplicaPlacement=%s", var.filer.default_replica_placement),
format("-dirListLimit=%d", var.filer.dir_list_limit),
format("-defaultStoreDir=%s", coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)),
],
var.filer.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.filer.nodes[k].grpc_port)] : [],
var.filer.max_mb != null ? [format("-maxMB=%d", var.filer.max_mb)] : [],
var.filer.disable_dir_listing ? ["-disableDirListing"] : [],
var.filer.disable_http ? ["-disableHttp"] : [],
var.filer.encrypt_volume_data ? ["-encryptVolumeData"] : [],
var.filer.rack != "" ? [format("-rack=%s", var.filer.rack)] : [],
var.filer.data_center != "" ? [format("-dataCenter=%s", var.filer.data_center)] : [],
var.filer.filer_group != "" ? [format("-filerGroup=%s", var.filer.filer_group)] : [],
var.filer.s3.enabled ? [
"-s3",
format("-s3.port=%d", var.filer.s3.port),
format("-s3.config=%s", var.filer.s3.config_path),
] : [],
(var.filer.s3.enabled && var.filer.s3.https_port > 0) ? [format("-s3.port.https=%d", var.filer.s3.https_port)] : [],
(var.filer.s3.enabled && var.filer.s3.domain_name != "") ? [format("-s3.domainName=%s", var.filer.s3.domain_name)] : [],
local.metrics_flags_filer,
var.filer.extra_args,
)
}
} : {}
# ===========================================================================
# S3 standalone nodes
# ===========================================================================
s3_keys = sort(keys(var.s3.nodes))
s3_target = var.s3.filer_address != "" ? var.s3.filer_address : local.first_filer
s3_std_files = merge(local.security_files, { (var.s3.config_path) = local.s3_config_json })
s3_nodes = var.s3.enabled ? {
for k in local.s3_keys : "s3-${k}" => {
role = "s3"
name = "s3-${k}"
address = var.s3.nodes[k].address
disk_mounts = []
data_dir = ""
data_dirs = []
ports = {
http = coalesce(var.s3.nodes[k].port, var.s3.port)
metrics = var.s3.metrics_port
}
env = local.cluster_env
config_files = local.s3_std_files
argv = concat(
local.global_flags,
["s3"],
[
format("-port=%d", coalesce(var.s3.nodes[k].port, var.s3.port)),
format("-ip.bind=%s", var.s3.ip_bind),
format("-filer=%s", local.s3_target),
format("-config=%s", var.s3.config_path),
],
var.s3.https_port > 0 ? [format("-port.https=%d", var.s3.https_port)] : [],
var.s3.iceberg_port != null ? [format("-port.iceberg=%d", var.s3.iceberg_port)] : [],
var.s3.domain_name != "" ? [format("-domainName=%s", var.s3.domain_name)] : [],
var.s3.audit_log_config_path != "" ? [format("-auditLogConfig=%s", var.s3.audit_log_config_path)] : [],
var.s3.cert_file != "" ? [format("-cert.file=%s", var.s3.cert_file)] : [],
var.s3.key_file != "" ? [format("-key.file=%s", var.s3.key_file)] : [],
var.s3.cacert_file != "" ? [format("-cacert.file=%s", var.s3.cacert_file)] : [],
var.s3.verify_client_cert ? ["-tlsVerifyClientCert"] : [],
local.metrics_flags_s3,
var.s3.extra_args,
)
}
} : {}
# ===========================================================================
# ALL-IN-ONE nodes (weed server)
# ===========================================================================
aio_keys = sort(keys(var.all_in_one.nodes))
aio_s3_files = var.all_in_one.s3.enabled ? { (var.all_in_one.s3.config_path) = local.s3_config_json } : {}
aio_nodes = var.all_in_one.enabled ? {
for k in local.aio_keys : "all-in-one-${k}" => {
role = "server"
name = "all-in-one-${k}"
address = var.all_in_one.nodes[k].address
disk_mounts = var.all_in_one.nodes[k].disk_mounts
data_dir = coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)
data_dirs = [coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)]
ports = {
http = var.all_in_one.master_port
metrics = var.all_in_one.metrics_port
}
env = local.cluster_env
config_files = merge(local.security_files, local.aio_s3_files)
argv = concat(
local.global_flags,
["server"],
[
format("-dir=%s", coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)),
format("-ip=%s", var.all_in_one.nodes[k].address),
format("-ip.bind=%s", var.all_in_one.ip_bind),
"-master",
format("-master.port=%d", var.all_in_one.master_port),
format("-master.peers=%s:%d", var.all_in_one.nodes[k].address, var.all_in_one.master_port),
format("-master.defaultReplication=%s", var.all_in_one.default_replication),
format("-master.volumeSizeLimitMB=%d", var.all_in_one.volume_size_limit_mb),
"-volume",
format("-volume.port=%d", var.all_in_one.volume_port),
"-filer",
format("-filer.port=%d", var.all_in_one.filer_port),
format("-idleTimeout=%d", var.all_in_one.idle_timeout),
],
var.all_in_one.disable_http ? [format("-disableHttp=%t", var.all_in_one.disable_http)] : [],
var.all_in_one.s3.enabled ? [
"-s3",
format("-s3.port=%d", var.all_in_one.s3.port),
format("-s3.config=%s", var.all_in_one.s3.config_path),
] : [],
(var.all_in_one.s3.enabled && var.all_in_one.s3.domain_name != "") ? [format("-s3.domainName=%s", var.all_in_one.s3.domain_name)] : [],
var.monitoring_enabled ? [format("-metricsPort=%d", var.all_in_one.metrics_port)] : [],
var.all_in_one.extra_args,
)
}
} : {}
# ===========================================================================
# Merge + render systemd unit and cloud-init per node
# ===========================================================================
nodes_base = merge(local.master_nodes, local.volume_nodes, local.filer_nodes, local.s3_nodes, local.aio_nodes)
# Disk-mount script per node (empty unless the node declares disk_mounts).
mount_scripts = {
for name, n in local.nodes_base : name => length(n.disk_mounts) > 0 ? templatefile("${path.module}/templates/mount-disks.sh.tftpl", {
run_as_user = var.hardening.run_as_user
mounts = n.disk_mounts
}) : ""
}
nodes = {
for name, n in local.nodes_base : name => merge(n, {
exec_start = "${var.weed_binary} ${join(" ", n.argv)}"
# secret_files: the secret-bearing config a wrapper should deliver from a
# secret store when render_secret_files=false (else they go in cloud-init).
secret_files = n.config_files
mount_script = local.mount_scripts[name]
file_modes = { for p in keys(n.config_files) : p => "0600" }
systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", {
role = n.role
name = n.name
run_as_user = var.hardening.run_as_user
exec_start = "${var.weed_binary} ${join(" ", n.argv)}"
no_new_privileges = var.hardening.no_new_privileges
protect_system = var.hardening.protect_system
cap_drop_all = var.hardening.cap_drop_all
read_write_paths = n.data_dirs
requires_mounts_for = n.data_dirs
env_file = var.env_file
environment = n.env
})
cloud_init = templatefile("${path.module}/templates/cloud-init.yaml.tftpl", {
role = n.role
name = n.name
run_as_user = var.hardening.run_as_user
config_files = var.render_secret_files ? n.config_files : {}
file_modes = { for p in keys(n.config_files) : p => "0600" }
pre_runcmd = [for p in n.data_dirs : "install -d -o ${var.hardening.run_as_user} -g ${var.hardening.run_as_user} ${p}"]
mount_script = local.mount_scripts[name]
fetch_script = var.boot_fetch_script
systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", {
role = n.role
name = n.name
run_as_user = var.hardening.run_as_user
exec_start = "${var.weed_binary} ${join(" ", n.argv)}"
no_new_privileges = var.hardening.no_new_privileges
protect_system = var.hardening.protect_system
cap_drop_all = var.hardening.cap_drop_all
read_write_paths = n.data_dirs
requires_mounts_for = n.data_dirs
env_file = var.env_file
environment = n.env
})
})
})
}
}
-32
View File
@@ -1,32 +0,0 @@
output "nodes" {
description = "Per-node rendered artifacts keyed by node name. Each value: role, name, address, ports, data_dir(s), argv (list, after the weed binary), exec_start, env, config_files, systemd_unit, cloud_init."
value = local.nodes
sensitive = true # config_files/env may carry JWT keys or S3 identities
}
output "master_peers" {
description = "Comma-separated master peer list (address:port), as passed to -peers / -mserver / -master."
value = local.master_peers
}
output "first_filer" {
description = "address:port of the lexically-first filer node (default S3 target)."
value = local.first_filer
}
output "node_names" {
description = "List of rendered node names (non-sensitive)."
value = sort(keys(local.nodes))
}
output "cloud_init_by_node" {
description = "Map of node name -> rendered cloud-init user_data (for the per-cloud wrappers)."
value = { for name, n in local.nodes : name => n.cloud_init }
sensitive = true
}
output "secret_files_by_node" {
description = "Map of node name -> {path => content} of secret-bearing files (security.toml, S3 identity JSON). Wrappers push these to a secret store and fetch them at boot when render_secret_files=false."
value = { for name, n in local.nodes : name => n.secret_files }
sensitive = true
}
@@ -1,44 +0,0 @@
#cloud-config
# Rendered by the SeaweedFS Terraform core module for node ${name} (role ${role}).
# When render_secret_files=false, secret-bearing files are NOT here; they are
# fetched at boot by fetch-secrets.sh from the cloud secret store.
users:
- name: ${run_as_user}
system: true
shell: /usr/sbin/nologin
write_files:
%{ for path, content in config_files ~}
- path: ${path}
owner: "${run_as_user}:${run_as_user}"
permissions: '${lookup(file_modes, path, "0644")}'
content: |
${indent(6, content)}
%{ endfor ~}
%{ if mount_script != "" ~}
- path: /opt/seaweedfs/mount-disks.sh
permissions: '0755'
content: |
${indent(6, mount_script)}
%{ endif ~}
%{ if fetch_script != "" ~}
- path: /opt/seaweedfs/fetch-secrets.sh
permissions: '0755'
content: |
${indent(6, fetch_script)}
%{ endif ~}
- path: /etc/systemd/system/weed-${role}.service
permissions: '0644'
content: |
${indent(6, systemd_unit)}
runcmd:
%{ if mount_script != "" ~}
- /opt/seaweedfs/mount-disks.sh
%{ endif ~}
%{ if fetch_script != "" ~}
- /opt/seaweedfs/fetch-secrets.sh
%{ endif ~}
%{ for cmd in pre_runcmd ~}
- ${cmd}
%{ endfor ~}
- systemctl daemon-reload
- systemctl enable --now weed-${role}.service
@@ -1,73 +0,0 @@
#!/usr/bin/env bash
# Rendered by the SeaweedFS Terraform core module. Formats (only if blank) and
# mounts each data disk, persisting by UUID to /etc/fstab. Safe to re-run.
set -euo pipefail
run_as_user="${run_as_user}"
# Base block device backing "/" so auto-discovery never touches the root disk.
root_src="$(findmnt -no SOURCE / 2>/dev/null || true)"
root_base="$(lsblk -no pkname "$root_src" 2>/dev/null || true)"
[ -n "$root_base" ] && root_base="/dev/$root_base"
is_mounted() { findmnt -rno TARGET "$1" >/dev/null 2>&1; }
# Pick the first unmounted, non-root whole disk (single-data-disk node).
discover_dev() {
local d base
for d in /dev/nvme*n1 /dev/xvd[b-z] /dev/sd[b-z] /dev/vd[b-z]; do
[ -b "$d" ] || continue
[ "$d" = "$root_src" ] && continue
[ "$d" = "$root_base" ] && continue
base="/dev/$(lsblk -no pkname "$d" 2>/dev/null || true)"
[ "$base" = "$root_base" ] && continue
is_mounted "$d" && continue
echo "$d"; return 0
done
return 1
}
# Wait up to ~120s for one of the explicit candidate devices to appear.
wait_dev() {
local i d
for i in $(seq 1 60); do
for d in "$@"; do [ -b "$d" ] && { echo "$d"; return 0; }; done
sleep 2
done
return 1
}
mount_one() {
local mountpoint="$1" fstype="$2"; shift 2
local dev=""
if [ "$#" -gt 0 ]; then
dev="$(wait_dev "$@" || true)"
fi
if [ -z "$dev" ]; then
dev="$(discover_dev || true)"
fi
if [ -z "$dev" ]; then
echo "mount-disks: no device found for $mountpoint" >&2
return 1
fi
if ! blkid "$dev" >/dev/null 2>&1; then
echo "mount-disks: formatting $dev as $fstype"
"mkfs.$fstype" -q "$dev"
fi
mkdir -p "$mountpoint"
local uuid; uuid="$(blkid -s UUID -o value "$dev")"
if [ -z "$uuid" ]; then
echo "mount-disks: failed to read a filesystem UUID for $dev" >&2
return 1
fi
if ! grep -q "$uuid" /etc/fstab; then
printf 'UUID=%s %s %s defaults,nofail 0 2\n' "$uuid" "$mountpoint" "$fstype" >> /etc/fstab
fi
is_mounted "$mountpoint" || mount "$mountpoint"
chown -R "$run_as_user:$run_as_user" "$mountpoint"
echo "mount-disks: $dev mounted at $mountpoint"
}
%{ for m in mounts ~}
mount_one "${m.mountpoint}" "${m.fstype}"${length(m.devices) > 0 ? " ${join(" ", m.devices)}" : ""}
%{ endfor ~}
@@ -1,53 +0,0 @@
# SeaweedFS security.toml rendered by the Terraform core module.
# Rendered when enable_security OR any non-default JWT signing option is set.
# Presence of this file does NOT imply mTLS: the [grpc] TLS block is emitted
# only under enable_security; a JWT-only config carries [jwt.*] and no TLS.
%{ if enable_security ~}
[grpc]
ca = "${cert_dir}/ca/tls.crt"
# Peer-identity authorization: without an allow-list, mTLS collapses to
# "trusted the CA" with no authZ. Set allowed_wildcard_domain / commonNames.
%{ if allowed_wildcard_domain != "" ~}
allowed_wildcard_domain = "${allowed_wildcard_domain}"
%{ endif ~}
[grpc.master]
cert = "${cert_dir}/master/tls.crt"
key = "${cert_dir}/master/tls.key"
%{ if allowed_common_names != "" ~}
allowed_commonNames = "${allowed_common_names}"
%{ endif ~}
[grpc.volume]
cert = "${cert_dir}/volume/tls.crt"
key = "${cert_dir}/volume/tls.key"
[grpc.filer]
cert = "${cert_dir}/filer/tls.crt"
key = "${cert_dir}/filer/tls.key"
[grpc.client]
cert = "${cert_dir}/client/tls.crt"
key = "${cert_dir}/client/tls.key"
%{ endif ~}
%{ if jwt_signing_key != "" ~}
[jwt.signing]
key = "${jwt_signing_key}"
%{ endif ~}
%{ if jwt_signing_read_key != "" ~}
[jwt.signing.read]
key = "${jwt_signing_read_key}"
%{ endif ~}
%{ if jwt_filer_signing_key != "" ~}
[jwt.filer_signing]
key = "${jwt_filer_signing_key}"
%{ endif ~}
%{ if jwt_filer_signing_read_key != "" ~}
[jwt.filer_signing.read]
key = "${jwt_filer_signing_read_key}"
%{ endif ~}
@@ -1,39 +0,0 @@
[Unit]
Description=SeaweedFS ${role} (${name})
Documentation=https://github.com/seaweedfs/seaweedfs/wiki
After=network-online.target
Wants=network-online.target
%{ for path in requires_mounts_for ~}
RequiresMountsFor=${path}
%{ endfor ~}
[Service]
Type=simple
User=${run_as_user}
Group=${run_as_user}
ExecStart=${exec_start}
Restart=always
RestartSec=5
LimitNOFILE=1048576
TimeoutStopSec=60
# Hardening (OpenShift SCC analogs; relax via the hardening variable)
NoNewPrivileges=${no_new_privileges}
%{ if protect_system ~}
ProtectSystem=strict
ProtectHome=true
PrivateTmp=true
%{ for path in read_write_paths ~}
ReadWritePaths=${path}
%{ endfor ~}
%{ endif ~}
%{ if cap_drop_all ~}
CapabilityBoundingSet=
AmbientCapabilities=
%{ endif ~}
EnvironmentFile=-${env_file}
%{ for k, v in environment ~}
Environment="${k}=${v}"
%{ endfor ~}
[Install]
WantedBy=multi-user.target
@@ -1,234 +0,0 @@
# Plan-level tests for the core renderer. No cloud credentials required.
# cd terraform/modules/core && tofu test
variables {
weed_binary = "/usr/bin/weed"
}
# ---------------------------------------------------------------------------
run "master_peers_and_count" {
command = plan
variables {
master = {
nodes = {
m0 = { address = "10.0.0.10" }
m1 = { address = "10.0.0.11" }
m2 = { address = "10.0.0.12" }
}
}
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = output.master_peers == "10.0.0.10:9333,10.0.0.11:9333,10.0.0.12:9333"
error_message = "master -peers list is wrong"
}
assert {
condition = length(output.node_names) == 3
error_message = "expected exactly 3 master nodes"
}
}
# ---------------------------------------------------------------------------
run "volume_uses_mserver_not_master" {
command = plan
variables {
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { nodes = { v0 = { address = "10.0.0.20", rack = "rack-a", data_center = "dc1" } } }
filer = { enabled = false }
}
assert {
condition = contains(output.nodes["volume-v0"].argv, "-mserver=10.0.0.10:9333")
error_message = "volume must pass the master list via -mserver (verified flag), not -master"
}
assert {
condition = !contains(output.nodes["volume-v0"].argv, "-master=10.0.0.10:9333")
error_message = "volume should not use -master for the master list"
}
assert {
condition = contains(output.nodes["volume-v0"].argv, "-rack=rack-a") && contains(output.nodes["volume-v0"].argv, "-dataCenter=dc1")
error_message = "per-node rack/dataCenter must render"
}
}
# ---------------------------------------------------------------------------
run "filer_uses_master_and_store_dir" {
command = plan
variables {
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { enabled = false }
filer = { nodes = { f0 = { address = "10.0.0.30", data_dir = "/srv/filer" } } }
}
assert {
condition = contains(output.nodes["filer-f0"].argv, "-master=10.0.0.10:9333")
error_message = "filer must use -master for the master list"
}
assert {
condition = contains(output.nodes["filer-f0"].argv, "-defaultStoreDir=/srv/filer")
error_message = "filer leveldb2 store dir must render via -defaultStoreDir"
}
}
# ---------------------------------------------------------------------------
run "metrics_gated_off_by_default" {
command = plan
variables {
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = length([for a in output.nodes["master-m0"].argv : a if startswith(a, "-metricsPort")]) == 0
error_message = "metrics port must NOT be bound when monitoring is disabled"
}
}
run "metrics_on_when_enabled" {
command = plan
variables {
monitoring_enabled = true
master = { nodes = { m0 = { address = "10.0.0.10" } }, metrics_port = 9327 }
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = contains(output.nodes["master-m0"].argv, "-metricsPort=9327")
error_message = "metrics port must bind when monitoring is enabled"
}
}
# ---------------------------------------------------------------------------
run "security_toml_off_by_default" {
command = plan
variables {
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = length(keys(output.nodes["master-m0"].config_files)) == 0
error_message = "security.toml must not render when security is off and no JWT keys set"
}
}
run "security_toml_on_with_mtls" {
command = plan
variables {
enable_security = true
security = {
allowed_wildcard_domain = ".seaweedfs.internal"
jwt_signing_key = "test-signing-key-not-a-real-secret-0123456789"
}
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml")
error_message = "security.toml must render when enable_security is true"
}
}
# ---------------------------------------------------------------------------
run "all_in_one_renders_server" {
command = plan
variables {
master = { enabled = false }
volume = { enabled = false }
filer = { enabled = false }
all_in_one = {
enabled = true
nodes = { a0 = { address = "10.0.0.40" } }
s3 = { enabled = true }
}
}
assert {
condition = contains(output.nodes["all-in-one-a0"].argv, "server")
error_message = "all-in-one must invoke the weed server subcommand"
}
assert {
condition = contains(output.nodes["all-in-one-a0"].argv, "-s3") && contains(output.nodes["all-in-one-a0"].argv, "-s3.port=8333")
error_message = "all-in-one S3 must render when enabled"
}
}
# ---------------------------------------------------------------------------
run "security_toml_mtls_only_no_jwt" {
command = plan
variables {
enable_security = true
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml")
error_message = "security.toml must render for mTLS even without JWT keys"
}
assert {
condition = !strcontains(output.nodes["master-m0"].config_files["/etc/seaweedfs/security.toml"], "[jwt.signing]")
error_message = "the jwt.signing block must be omitted when no signing key is set"
}
}
run "mount_fetch_and_secret_delivery" {
command = plan
variables {
enable_security = true
render_secret_files = false
boot_fetch_script = "#!/usr/bin/env bash\necho fetch\n"
security = {
jwt_signing_key = "test-signing-key-not-a-real-secret-0123456789"
}
master = { nodes = { m0 = { address = "10.0.0.10", disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }] } } }
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/mount-disks.sh")
error_message = "cloud-init must write+run the mount script when disk_mounts is set"
}
assert {
condition = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/fetch-secrets.sh")
error_message = "cloud-init must write+run the fetch script when boot_fetch_script is set"
}
assert {
condition = !strcontains(output.nodes["master-m0"].cloud_init, "[jwt.signing]")
error_message = "security.toml must NOT be inlined in cloud-init when render_secret_files=false"
}
assert {
condition = contains(keys(output.nodes["master-m0"].secret_files), "/etc/seaweedfs/security.toml")
error_message = "secret_files must expose security.toml for secret-store delivery"
}
}
run "mount_script_absent_without_disks" {
command = plan
variables {
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { enabled = false }
filer = { enabled = false }
}
assert {
condition = !strcontains(output.nodes["master-m0"].cloud_init, "mount-disks.sh")
error_message = "no mount script should render when the node declares no disk_mounts"
}
}
run "s3_standalone_targets_filer" {
command = plan
variables {
master = { nodes = { m0 = { address = "10.0.0.10" } } }
volume = { enabled = false }
filer = { nodes = { f0 = { address = "10.0.0.30" } } }
s3 = { enabled = true, nodes = { s0 = { address = "10.0.0.50" } } }
}
assert {
condition = contains(output.nodes["s3-s0"].argv, "-filer=10.0.0.30:8888")
error_message = "standalone S3 must point -filer at the first filer node"
}
assert {
condition = contains(output.nodes["s3-s0"].argv, "-config=/etc/seaweedfs/s3_config.json")
error_message = "standalone S3 must reference the rendered config path"
}
}
-318
View File
@@ -1,318 +0,0 @@
# =============================================================================
# SeaweedFS Terraform core module - input variables
#
# This module is a PURE RENDERER. It creates zero cloud resources. It takes a
# per-node address map plus configuration and emits, per node, the rendered
# `weed` argv, systemd unit, cloud-init, and config files. Both the per-cloud
# infra wrappers (consume cloud_init/systemd_unit) and the local test harness
# (consume argv/config_files) read its `nodes` output.
#
# Flag names below are verified against the real `weed` binary, not the chart.
# Notably: volume uses -mserver for the master list; gRPC is -port.grpc
# (auto = http+10000); minFreeSpacePercent is a string.
# =============================================================================
variable "weed_binary" {
description = "Path to the weed executable on the target host."
type = string
default = "/usr/bin/weed"
}
variable "cluster_name" {
description = "Logical cluster name (WEED_CLUSTER_DEFAULT)."
type = string
default = "sw"
}
variable "log_level" {
description = "Global glog verbosity (-v)."
type = number
default = 1
}
variable "log_to_stderr" {
description = "Log to stderr (journald) instead of -logdir. Recommended on systemd."
type = bool
default = true
}
variable "monitoring_enabled" {
description = "Gate the -metricsPort/-metrics.address flags. When false the metrics port is not bound, matching the chart."
type = bool
default = false
}
variable "enable_security" {
description = "Emit the security.toml [grpc] mTLS block and reference per-component certs. Independent of JWT."
type = bool
default = false
}
# ----------------------------------------------------------------------------
# Security / secrets (the core only references paths + renders security.toml;
# cert/secret material is generated by the wrapper, never defaulted here).
# ----------------------------------------------------------------------------
variable "security" {
description = "mTLS + JWT signing configuration. Secret values must be supplied (never defaulted in module code)."
type = object({
cert_dir = optional(string, "/usr/local/share/ca-certificates")
allowed_wildcard_domain = optional(string, "")
allowed_common_names = optional(string, "")
jwt_signing_key = optional(string, "")
jwt_signing_read_key = optional(string, "")
jwt_filer_signing_key = optional(string, "")
jwt_filer_signing_read_key = optional(string, "")
})
default = {}
sensitive = true
}
variable "hardening" {
description = "systemd hardening directives (OpenShift SCC analogs). Relax per field if needed."
type = object({
run_as_user = optional(string, "seaweedfs")
no_new_privileges = optional(bool, true)
protect_system = optional(bool, true)
cap_drop_all = optional(bool, true)
})
default = {}
}
variable "env_file" {
description = "Path to the systemd EnvironmentFile holding secret env (DB creds, S3 admin key) fetched at boot. Optional (-prefixed in the unit, so a missing file is tolerated)."
type = string
default = "/etc/seaweedfs/weed.env"
}
variable "render_secret_files" {
description = <<-EOT
When true (default), secret-bearing config (security.toml, S3 identity JSON)
is written directly into cloud-init user_data. Convenient for the local test
harness and simple deployments. Set to false to keep secrets OUT of user_data
(and out of the cloud metadata service): the files are then exposed only via
the secret_files_by_node output, and the wrapper is expected to deliver them
from a secret store via boot_fetch_script.
EOT
type = bool
default = true
}
variable "boot_fetch_script" {
description = "Optional shell script written to /opt/seaweedfs/fetch-secrets.sh (root, 0755) and run before the weed unit starts. Used by wrappers to pull certs/secrets from a cloud secret store at boot."
type = string
default = ""
}
# ----------------------------------------------------------------------------
# Master (Raft quorum). Keyed `nodes` map -> for_each, never count.
# ----------------------------------------------------------------------------
variable "master" {
description = "Master tier. nodes is a map keyed by stable node id (m0/m1/m2)."
type = object({
enabled = optional(bool, true)
nodes = optional(map(object({
address = string
port = optional(number)
grpc_port = optional(number)
data_dir = optional(string)
disk_mounts = optional(list(object({
mountpoint = string
fstype = optional(string, "ext4")
devices = optional(list(string), [])
})), [])
})), {})
port = optional(number, 9333)
grpc_port = optional(number, null) # null => weed auto = port+10000
metrics_port = optional(number, 9327)
metrics_ip = optional(string, "")
ip_bind = optional(string, "0.0.0.0")
data_dir = optional(string, "/var/lib/seaweedfs/master")
default_replication = optional(string, "000")
volume_size_limit_mb = optional(number, 1000)
volume_preallocate = optional(bool, false)
garbage_threshold = optional(number, null)
disable_http = optional(bool, false)
raft_hashicorp = optional(bool, false)
resume_state = optional(bool, true) # deliberate deviation: binary default false
election_timeout = optional(string, "10s")
heartbeat_interval = optional(string, "300ms")
white_list = optional(string, "")
extra_args = optional(list(string), [])
})
default = {}
}
# ----------------------------------------------------------------------------
# Volume (stateful disk owners). Keyed `nodes` map.
# ----------------------------------------------------------------------------
variable "volume" {
description = "Volume tier. Each node owns one or more data dirs (-> -dir / -max)."
type = object({
enabled = optional(bool, true)
nodes = optional(map(object({
address = string
port = optional(number)
grpc_port = optional(number)
public_url = optional(string)
rack = optional(string)
data_center = optional(string)
data_dirs = optional(list(object({
path = string
max_volumes = optional(number, 0) # 0 => auto on non-windows
})))
idx_dir = optional(string)
disk_mounts = optional(list(object({
mountpoint = string
fstype = optional(string, "ext4")
devices = optional(list(string), [])
})), [])
})), {})
port = optional(number, 8080)
metrics_port = optional(number, 9327)
metrics_ip = optional(string, "")
ip_bind = optional(string, "0.0.0.0")
data_dirs = optional(list(object({
path = string
max_volumes = optional(number, 0)
})), [{ path = "/data", max_volumes = 0 }])
idx_dir = optional(string, "")
read_mode = optional(string, "proxy")
compaction_mbps = optional(number, 50)
min_free_space_percent = optional(string, "1")
file_size_limit_mb = optional(number, null)
index = optional(string, "memory")
images_fix_orientation = optional(bool, false)
white_list = optional(string, "")
extra_args = optional(list(string), [])
})
default = {}
}
# ----------------------------------------------------------------------------
# Filer (+ optional embedded S3). Keyed `nodes` map.
# ----------------------------------------------------------------------------
variable "filer" {
description = "Filer tier. Default leveldb2-replicated HA (each filer has its own store)."
type = object({
enabled = optional(bool, true)
nodes = optional(map(object({
address = string
port = optional(number)
grpc_port = optional(number)
data_dir = optional(string)
disk_mounts = optional(list(object({
mountpoint = string
fstype = optional(string, "ext4")
devices = optional(list(string), [])
})), [])
})), {})
port = optional(number, 8888)
metrics_port = optional(number, 9327)
metrics_ip = optional(string, "")
ip_bind = optional(string, "0.0.0.0")
data_dir = optional(string, "/var/lib/seaweedfs/filer") # -defaultStoreDir for leveldb2
default_replica_placement = optional(string, "000")
dir_list_limit = optional(number, 100000)
max_mb = optional(number, null)
disable_dir_listing = optional(bool, false)
disable_http = optional(bool, false)
encrypt_volume_data = optional(bool, false)
rack = optional(string, "")
data_center = optional(string, "")
filer_group = optional(string, "")
extra_env = optional(map(string), {})
extra_args = optional(list(string), [])
s3 = optional(object({
enabled = optional(bool, false)
port = optional(number, 8333)
https_port = optional(number, 0)
domain_name = optional(string, "")
config_path = optional(string, "/etc/seaweedfs/s3_config.json")
}), {})
})
default = {}
}
# ----------------------------------------------------------------------------
# S3 gateway (standalone). Keyed `nodes` map (stateless, but kept keyed for
# parity; wrappers may instead drive an ASG).
# ----------------------------------------------------------------------------
variable "s3" {
description = "Standalone S3 gateway tier."
type = object({
enabled = optional(bool, false)
nodes = optional(map(object({
address = string
port = optional(number)
grpc_port = optional(number)
})), {})
port = optional(number, 8333)
https_port = optional(number, 0)
iceberg_port = optional(number, null)
metrics_port = optional(number, 9327)
ip_bind = optional(string, "0.0.0.0")
domain_name = optional(string, "")
filer_address = optional(string, "") # override; else first filer node
config_path = optional(string, "/etc/seaweedfs/s3_config.json")
audit_log_config_path = optional(string, "")
cert_file = optional(string, "")
key_file = optional(string, "")
cacert_file = optional(string, "")
verify_client_cert = optional(bool, false)
extra_args = optional(list(string), [])
})
default = {}
}
# ----------------------------------------------------------------------------
# S3 identities -> rendered into the s3 config JSON (non-admin only; the admin
# credential is delivered via EnvironmentFile, never the JSON).
# ----------------------------------------------------------------------------
variable "s3_identities" {
description = "Non-admin S3 identities rendered into the config JSON. Admin key goes in the EnvironmentFile, not here."
type = list(object({
name = string
access_key = string
secret_key = string
actions = list(string)
}))
default = []
sensitive = true
}
# ----------------------------------------------------------------------------
# All-in-one (single `weed server` process).
# ----------------------------------------------------------------------------
variable "all_in_one" {
description = "All-in-one tier (weed server). Mutually exclusive with the distributed tiers."
type = object({
enabled = optional(bool, false)
nodes = optional(map(object({
address = string
data_dir = optional(string)
disk_mounts = optional(list(object({
mountpoint = string
fstype = optional(string, "ext4")
devices = optional(list(string), [])
})), [])
})), {})
data_dir = optional(string, "/data")
ip_bind = optional(string, "0.0.0.0")
master_port = optional(number, 9333)
volume_port = optional(number, 8080)
filer_port = optional(number, 8888)
default_replication = optional(string, "000")
volume_size_limit_mb = optional(number, 1000)
idle_timeout = optional(number, 30)
metrics_port = optional(number, 9324)
disable_http = optional(bool, false)
s3 = optional(object({
enabled = optional(bool, false)
port = optional(number, 8333)
config_path = optional(string, "/etc/seaweedfs/s3/s3_config.json")
domain_name = optional(string, "")
}), {})
extra_args = optional(list(string), [])
})
default = {}
}
-6
View File
@@ -1,6 +0,0 @@
terraform {
# 1.3+ is required for optional() object attributes with defaults.
# The published wrappers pin a single honest floor;
# the core itself only needs language features available since 1.3.
required_version = ">= 1.3.0"
}
-68
View File
@@ -1,68 +0,0 @@
locals {
components = ["master", "volume", "filer", "client"]
ip_sans = distinct(concat(["127.0.0.1"], var.ip_sans))
}
# ---- CA ---------------------------------------------------------------------
resource "tls_private_key" "ca" {
algorithm = var.key_algorithm
ecdsa_curve = var.ecdsa_curve
rsa_bits = var.rsa_bits
}
resource "tls_self_signed_cert" "ca" {
private_key_pem = tls_private_key.ca.private_key_pem
is_ca_certificate = true
subject {
common_name = "SeaweedFS CA"
organization = "SeaweedFS"
}
validity_period_hours = var.ca_validity_hours
allowed_uses = ["cert_signing", "crl_signing", "digital_signature"]
}
# ---- per-component leaf certs (distinct CN per component) --------------------
resource "tls_private_key" "component" {
for_each = toset(local.components)
algorithm = var.key_algorithm
ecdsa_curve = var.ecdsa_curve
rsa_bits = var.rsa_bits
}
resource "tls_cert_request" "component" {
for_each = toset(local.components)
private_key_pem = tls_private_key.component[each.key].private_key_pem
subject {
# Distinct CN per component so peer-identity authZ (allowed_wildcard_domain
# / allowed_commonNames) is meaningful. Do NOT use one CN for all.
common_name = "${each.key}.${var.internal_domain}"
organization = "SeaweedFS"
}
dns_names = distinct(concat(
["${each.key}.${var.internal_domain}", "localhost"],
var.extra_dns_sans,
))
ip_addresses = local.ip_sans
}
resource "tls_locally_signed_cert" "component" {
for_each = toset(local.components)
cert_request_pem = tls_cert_request.component[each.key].cert_request_pem
ca_private_key_pem = tls_private_key.ca.private_key_pem
ca_cert_pem = tls_self_signed_cert.ca.cert_pem
validity_period_hours = var.cert_validity_hours
early_renewal_hours = var.cert_early_renewal_hours
allowed_uses = ["digital_signature", "key_agreement", "server_auth", "client_auth"]
}
# ---- JWT signing keys -------------------------------------------------------
resource "random_password" "jwt" {
for_each = var.generate_jwt ? toset(["signing", "signing_read", "filer_signing", "filer_signing_read"]) : toset([])
length = var.jwt_length
special = false # TOML-safe
}
-65
View File
@@ -1,65 +0,0 @@
output "ca_cert_pem" {
description = "CA certificate PEM."
value = tls_self_signed_cert.ca.cert_pem
}
output "certs" {
description = "Per-component { cert_pem, private_key_pem }."
value = {
for c in local.components : c => {
cert_pem = tls_locally_signed_cert.component[c].cert_pem
private_key_pem = tls_private_key.component[c].private_key_pem
}
}
sensitive = true
}
output "jwt" {
description = "Generated JWT keys (empty strings when generate_jwt = false)."
value = {
signing = var.generate_jwt ? random_password.jwt["signing"].result : ""
signing_read = var.generate_jwt ? random_password.jwt["signing_read"].result : ""
filer_signing = var.generate_jwt ? random_password.jwt["filer_signing"].result : ""
filer_signing_read = var.generate_jwt ? random_password.jwt["filer_signing_read"].result : ""
}
sensitive = true
}
# Ready to pass straight to the core module's `security` variable.
output "core_security" {
description = "Object shaped for the core module's `security` input (cert_dir, allowed_wildcard_domain, JWT keys)."
value = {
cert_dir = var.cert_dir
allowed_wildcard_domain = ".${var.internal_domain}"
allowed_common_names = ""
jwt_signing_key = var.generate_jwt ? random_password.jwt["signing"].result : ""
jwt_signing_read_key = var.generate_jwt ? random_password.jwt["signing_read"].result : ""
jwt_filer_signing_key = var.generate_jwt ? random_password.jwt["filer_signing"].result : ""
jwt_filer_signing_read_key = var.generate_jwt ? random_password.jwt["filer_signing_read"].result : ""
}
sensitive = true
}
# Non-sensitive manifest (key, on-host path, mode) so a wrapper can for_each
# over SSM/secret-store resources without tripping the sensitive-for_each rule.
output "secret_manifest" {
description = "List of { key, path, mode } for the CA + component cert/key files (no content)."
value = concat(
[{ key = "ca/tls.crt", path = "${var.cert_dir}/ca/tls.crt", mode = "0644" }],
flatten([for c in local.components : [
{ key = "${c}/tls.crt", path = "${var.cert_dir}/${c}/tls.crt", mode = "0644" },
{ key = "${c}/tls.key", path = "${var.cert_dir}/${c}/tls.key", mode = "0600" },
]]),
)
}
# Sensitive map keyed by the manifest key -> file content.
output "secret_contents" {
description = "Map of manifest key -> PEM content."
value = merge(
{ "ca/tls.crt" = tls_self_signed_cert.ca.cert_pem },
{ for c in local.components : "${c}/tls.crt" => tls_locally_signed_cert.component[c].cert_pem },
{ for c in local.components : "${c}/tls.key" => tls_private_key.component[c].private_key_pem },
)
sensitive = true
}
-89
View File
@@ -1,89 +0,0 @@
# =============================================================================
# SeaweedFS security material generator (cloud-agnostic).
#
# Generates the CA, per-component mTLS certs with DISTINCT CommonNames, and JWT
# signing keys. Outputs a `core_security` object ready to feed the core module's
# `security` variable, plus the raw PEMs for a wrapper to deliver via a secret
# store. NOTE: TF-generated secrets live in state; prefer Vault PKI for the CA
# in production.
# =============================================================================
variable "internal_domain" {
description = "Internal domain for component CNs (e.g. master.<domain>). Drives the allowed_wildcard_domain peer-auth check."
type = string
default = "seaweedfs.internal"
}
variable "ip_sans" {
description = "IP addresses to include as SANs on every component cert (all node private IPs + 127.0.0.1 is added automatically)."
type = list(string)
default = []
}
variable "extra_dns_sans" {
description = "Additional DNS SANs to include on every component cert."
type = list(string)
default = []
}
variable "key_algorithm" {
description = "CA/leaf key algorithm: ECDSA (default, modern) or RSA."
type = string
default = "ECDSA"
validation {
condition = contains(["ECDSA", "RSA"], var.key_algorithm)
error_message = "key_algorithm must be ECDSA or RSA."
}
}
variable "ecdsa_curve" {
description = "ECDSA curve when key_algorithm = ECDSA."
type = string
default = "P256"
}
variable "rsa_bits" {
description = "RSA key size when key_algorithm = RSA."
type = number
default = 3072
}
variable "ca_validity_hours" {
description = "CA certificate validity (default 10 years)."
type = number
default = 87600
}
variable "cert_validity_hours" {
description = "Component certificate validity. Short by default; certs hot-reload so reissue is cheap."
type = number
default = 72
}
variable "cert_early_renewal_hours" {
description = "Renew component certs this many hours before expiry."
type = number
default = 24
}
variable "cert_dir" {
description = "On-host directory where certs are placed; surfaced in core_security.cert_dir."
type = string
default = "/usr/local/share/ca-certificates"
}
variable "generate_jwt" {
description = "Generate JWT signing keys (signing, signing.read, filer_signing, filer_signing.read)."
type = bool
default = true
}
variable "jwt_length" {
description = "Length of generated JWT keys (>= 32 hardened)."
type = number
default = 40
validation {
condition = var.jwt_length >= 32
error_message = "jwt_length must be >= 32 (hardened minimum)."
}
}
-13
View File
@@ -1,13 +0,0 @@
terraform {
required_version = ">= 1.3.0"
required_providers {
tls = {
source = "hashicorp/tls"
version = ">= 4.0"
}
random = {
source = "hashicorp/random"
version = ">= 3.5"
}
}
}
-74
View File
@@ -1,74 +0,0 @@
# =============================================================================
# Local mTLS test harness: generate the CA + component certs + JWT keys with the
# security submodule, render security.toml with the core, then run a real
# master+volume+filer cluster with mTLS enabled on 127.0.0.1 and assert it works.
# Proves the generated security material is valid and accepted by weed.
# =============================================================================
terraform {
required_version = ">= 1.3.0"
}
variable "weed_binary" {
type = string
default = "/usr/bin/weed"
}
variable "workdir" {
type = string
default = "/tmp/seaweedfs-tftest-secure"
}
module "security" {
source = "../../modules/security"
internal_domain = "seaweedfs.internal"
ip_sans = ["127.0.0.1"]
cert_dir = "${var.workdir}/certs"
}
module "core" {
source = "../../modules/core"
weed_binary = var.weed_binary
enable_security = true
security = module.security.core_security
# We read security.toml back from the output and place it ourselves (under
# $HOME/.seaweedfs), so keep it in the rendered file set.
render_secret_files = true
# High, distinct ports so this never collides with a running SeaweedFS or the
# non-secure harness.
master = {
nodes = { m0 = { address = "127.0.0.1", port = 29336, data_dir = "${var.workdir}/master-m0" } }
election_timeout = "3s"
heartbeat_interval = "200ms"
}
volume = {
nodes = { v0 = { address = "127.0.0.1", port = 28081, data_dirs = [{ path = "${var.workdir}/volume-v0", max_volumes = 20 }] } }
}
filer = {
nodes = { f0 = { address = "127.0.0.1", port = 28889, data_dir = "${var.workdir}/filer-f0" } }
}
}
output "cluster" {
sensitive = true
value = {
for name, n in module.core.nodes : name => {
role = n.role
http_port = n.ports.http
data_dirs = n.data_dirs
argv = n.argv
env = n.env
}
}
}
output "security_toml" {
sensitive = true
value = module.core.secret_files_by_node["master-m0"]["/etc/seaweedfs/security.toml"]
}
output "certs" {
sensitive = true
value = [for m in module.security.secret_manifest : { path = m.path, mode = m.mode, content = module.security.secret_contents[m.key] }]
}
@@ -1,128 +0,0 @@
#!/usr/bin/env bash
# Generate mTLS material + security.toml with Terraform, then run a real
# master+volume+filer cluster with mTLS enabled and assert it works.
# ./run_local_secure.sh # render + run + assert + teardown
# KEEP=1 ./run_local_secure.sh
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
export PATH="/opt/homebrew/bin:$PATH"
TOFU="${TOFU:-tofu}"
WEED="${WEED:-$(go env GOPATH 2>/dev/null || echo "$HOME/go")/bin/weed}"
WORKDIR="${WORKDIR:-/tmp/seaweedfs-tftest-secure}"
LOGDIR="$WORKDIR/logs"
RUNDIR="$WORKDIR/run"
PASS=0
FAIL=0
ok() { echo " PASS: $1"; PASS=$((PASS + 1)); }
bad() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
info() { echo "==> $1"; }
cleanup() {
info "tearing down"
if [ -d "$RUNDIR" ]; then
for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill "$(cat "$pf")" 2>/dev/null; done
sleep 1
for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill -9 "$(cat "$pf")" 2>/dev/null; done
fi
}
info "cleaning $WORKDIR"
case "$WORKDIR" in "" | "/" | "$HOME") echo "refusing to delete '$WORKDIR'" >&2; exit 2 ;; esac
rm -rf "$WORKDIR"
mkdir -p "$LOGDIR" "$RUNDIR" "$WORKDIR/.seaweedfs"
[ -x "$WEED" ] || { echo "weed not found at $WEED" >&2; exit 2; }
info "generating certs + rendering config with OpenTofu"
cd "$HERE"
"$TOFU" init -backend=false -input=false -no-color >/dev/null 2>&1 || { echo "tofu init failed"; exit 2; }
if ! "$TOFU" apply -auto-approve -input=false -no-color \
-var "weed_binary=$WEED" -var "workdir=$WORKDIR" >"$LOGDIR/tofu.log" 2>&1; then
echo "tofu apply failed"; tail -30 "$LOGDIR/tofu.log"; exit 2
fi
# write certs to their on-host paths
"$TOFU" output -json certs | jq -c '.[]' | while IFS= read -r c; do
p="$(echo "$c" | jq -r '.path')"; m="$(echo "$c" | jq -r '.mode')"
mkdir -p "$(dirname "$p")"
echo "$c" | jq -r '.content' > "$p"
chmod "$m" "$p"
done
# place security.toml where weed searches ($HOME/.seaweedfs)
"$TOFU" output -raw security_toml > "$WORKDIR/.seaweedfs/security.toml"
info "security.toml + $(ls "$WORKDIR"/certs | wc -l | tr -d ' ') cert dirs written"
OUT="$("$TOFU" output -json cluster)"
# derive ports from the rendered config (high range; never hardcoded)
port_of() { echo "$OUT" | jq -r --arg r "$1" '.[] | select(.role==$r) | .http_port' | head -1; }
MPORT="$(port_of master)"; VPORT="$(port_of volume)"; FPORT="$(port_of filer)"
busy=""
for p in $(echo "$OUT" | jq -r '.[].http_port'); do
lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1 && busy="$busy $p"
done
[ -n "$busy" ] && { echo "Required port(s) already in use:$busy -- is another SeaweedFS running?" >&2; exit 3; }
[ "${KEEP:-0}" = "1" ] || trap cleanup EXIT INT TERM
# launch a node with HOME pointed at $WORKDIR so weed loads security.toml
launch() {
n="$1"
for d in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].data_dirs[]?'); do mkdir -p "$d"; done
ENVS=("HOME=$WORKDIR")
while IFS= read -r e; do [ -n "$e" ] && ENVS+=("$e"); done \
< <(echo "$OUT" | jq -r --arg n "$n" '.[$n].env | to_entries[] | "\(.key)=\(.value)"')
ARGV=()
while IFS= read -r a; do ARGV+=("$a"); done \
< <(echo "$OUT" | jq -r --arg n "$n" '.[$n].argv[]')
info "launching $n (mTLS)"
env "${ENVS[@]}" "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
echo "$!" > "$RUNDIR/$n.pid"
}
wait_http() {
url="$1"; t="${2:-30}"; i=0
while [ "$i" -lt "$t" ]; do
curl -fsS -o /dev/null --max-time 2 "$url" 2>/dev/null && return 0
i=$((i + 1)); sleep 1
done
return 1
}
launch master-m0
QOK=0; i=0
while [ "$i" -lt 40 ]; do
st="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT/cluster/status" 2>/dev/null)" || { i=$((i+1)); sleep 1; continue; }
[ "$(echo "$st" | jq -r '.IsLeader // false')" = "true" ] && { QOK=1; break; }
i=$((i + 1)); sleep 1
done
[ "$QOK" -eq 1 ] && ok "master elected leader under mTLS" || bad "master leader (mTLS)"
launch volume-v0
# registration (master gRPC over mTLS) confirms the volume joined the cluster
AOK=0; i=0
while [ "$i" -lt 40 ]; do
fid="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT/dir/assign" 2>/dev/null | jq -r '.fid // ""')"
[ -n "$fid" ] && { AOK=1; break; }
i=$((i + 1)); sleep 1
done
[ "$AOK" -eq 1 ] && ok "volume registered via mTLS gRPC (fid $fid)" || bad "volume registration over mTLS"
# /healthz flips to 200 after the first heartbeat completes (slower under mTLS)
wait_http "http://127.0.0.1:$VPORT/healthz" 60 && ok "volume /healthz up (mTLS)" || bad "volume /healthz"
launch filer-f0
wait_http "http://127.0.0.1:$FPORT/" 30 && ok "filer / up (mTLS)" || bad "filer /"
# [jwt.filer_signing] is active, so the filer requires a signed JWT for writes.
# An unsigned write MUST be rejected with 401 -- this proves the JWT signing key
# rendered into security.toml is enforced (positive security assertion).
echo "smoke" > "$WORKDIR/hello.txt"
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -X POST \
-F "file=@$WORKDIR/hello.txt" "http://127.0.0.1:$FPORT/smoke/hello.txt" 2>/dev/null)"
[ "$code" = "401" ] && ok "filer rejects unsigned write (HTTP 401) => JWT signing enforced" \
|| bad "filer JWT enforcement (expected 401, got HTTP $code)"
echo
info "RESULTS: $PASS passed, $FAIL failed"
[ "$FAIL" -eq 0 ]
-100
View File
@@ -1,100 +0,0 @@
# =============================================================================
# Local test harness: render a small SeaweedFS cluster with the core module
# and run it as real `weed` processes on 127.0.0.1 (no cloud, no docker).
#
# 3 masters (quorum) + 1 volume + 1 filer + 1 standalone S3, each on a distinct
# port. run_local_cluster.sh consumes the `cluster` output, launches the weed
# processes from the rendered argv, and asserts the cluster actually works.
# =============================================================================
terraform {
required_version = ">= 1.3.0"
}
variable "weed_binary" {
description = "Path to the weed executable used for the local cluster."
type = string
default = "/usr/bin/weed"
}
variable "workdir" {
description = "Scratch directory for per-node data dirs and config files."
type = string
default = "/tmp/seaweedfs-tftest"
}
module "core" {
source = "../../modules/core"
weed_binary = var.weed_binary
monitoring_enabled = false
enable_security = false
# High port range so the harness does not collide with a SeaweedFS cluster
# that may already be running on this dev machine (default 9333/8080/8888/8333).
master = {
nodes = {
m0 = { address = "127.0.0.1", port = 29333, data_dir = "${var.workdir}/master-m0" }
m1 = { address = "127.0.0.1", port = 29334, data_dir = "${var.workdir}/master-m1" }
m2 = { address = "127.0.0.1", port = 29335, data_dir = "${var.workdir}/master-m2" }
}
# Tighten election so a 3-node local quorum converges quickly.
election_timeout = "3s"
heartbeat_interval = "200ms"
}
volume = {
nodes = {
v0 = {
address = "127.0.0.1"
port = 28080
rack = "rack-a"
data_dirs = [{ path = "${var.workdir}/volume-v0", max_volumes = 20 }]
}
}
}
filer = {
nodes = {
f0 = { address = "127.0.0.1", port = 28888, data_dir = "${var.workdir}/filer-f0" }
}
}
s3 = {
enabled = true
nodes = { s0 = { address = "127.0.0.1", port = 28333 } }
# this weed build starts an Iceberg REST catalog on 8181 by default; disable
# it so the test does not collide with a cluster already using that port.
iceberg_port = 0
config_path = "${var.workdir}/s3_config.json"
}
# Anonymous identity so the smoke test can PUT/GET without sigv4 signing.
s3_identities = [{
name = "anonymous"
access_key = ""
secret_key = ""
actions = ["Admin", "Read", "Write", "List", "Tagging"]
}]
}
output "cluster" {
description = "Node specs consumed by run_local_cluster.sh (read via `tofu output -json`, which emits values even when sensitive)."
sensitive = true
value = {
for name, n in module.core.nodes : name => {
role = n.role
address = n.address
http_port = n.ports.http
data_dirs = n.data_dirs
argv = n.argv
env = n.env
config_files = n.config_files
exec_start = n.exec_start
}
}
}
output "master_peers" {
value = module.core.master_peers
}
-170
View File
@@ -1,170 +0,0 @@
#!/usr/bin/env bash
# Render a SeaweedFS cluster with the Terraform core module and run it as real
# `weed` processes locally, then assert it actually works. No cloud, no docker.
#
# ./run_local_cluster.sh # render + run + assert + teardown
# KEEP=1 ./run_local_cluster.sh # leave the cluster running after asserts
# WEED=/path/to/weed ./run_local_cluster.sh
#
# Ports come from the rendered config (high range, to avoid colliding with a
# SeaweedFS cluster already running on this machine). Exits non-zero on any
# failed assertion or if a required port is already in use.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
export PATH="/opt/homebrew/bin:$PATH"
TOFU="${TOFU:-tofu}"
WEED="${WEED:-$(go env GOPATH 2>/dev/null || echo "$HOME/go")/bin/weed}"
WORKDIR="${WORKDIR:-/tmp/seaweedfs-tftest}"
LOGDIR="$WORKDIR/logs"
RUNDIR="$WORKDIR/run"
PASS=0
FAIL=0
ok() { echo " PASS: $1"; PASS=$((PASS + 1)); }
bad() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
info() { echo "==> $1"; }
cleanup() {
info "tearing down"
if [ -d "$RUNDIR" ]; then
for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill "$(cat "$pf")" 2>/dev/null; done
sleep 1
for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill -9 "$(cat "$pf")" 2>/dev/null; done
fi
}
info "cleaning $WORKDIR"
case "$WORKDIR" in "" | "/" | "$HOME") echo "refusing to delete '$WORKDIR'" >&2; exit 2 ;; esac
rm -rf "$WORKDIR"
mkdir -p "$LOGDIR" "$RUNDIR"
[ -x "$WEED" ] || { echo "weed binary not found/executable at $WEED" >&2; exit 2; }
info "rendering cluster config with OpenTofu"
cd "$HERE"
"$TOFU" init -backend=false -input=false -no-color >/dev/null 2>&1 || { echo "tofu init failed"; exit 2; }
if ! "$TOFU" apply -auto-approve -input=false -no-color \
-var "weed_binary=$WEED" -var "workdir=$WORKDIR" >"$LOGDIR/tofu.log" 2>&1; then
echo "tofu apply failed; see $LOGDIR/tofu.log"; tail -30 "$LOGDIR/tofu.log"; exit 2
fi
OUT="$("$TOFU" output -json cluster)"
[ -n "$OUT" ] || { echo "empty cluster output"; exit 2; }
# ---- derive ports by role (never hardcoded) ---------------------------------
port_of() { echo "$OUT" | jq -r --arg r "$1" '.[] | select(.role==$r) | .http_port' | head -1; }
mports() { echo "$OUT" | jq -r '.[] | select(.role=="master") | .http_port' | sort; }
MPORT1="$(mports | head -1)"
VPORT="$(port_of volume)"; FPORT="$(port_of filer)"; SPORT="$(port_of s3)"
NMASTERS="$(mports | wc -l | tr -d ' ')"
# ---- refuse to run if a required port is already taken (foreign cluster) -----
busy=""
for p in $(echo "$OUT" | jq -r '.[].http_port'); do
lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1 && busy="$busy $p"
done
[ -n "$busy" ] && { echo "Required port(s) already in use:$busy -- is another SeaweedFS running?" >&2; exit 3; }
[ "${KEEP:-0}" = "1" ] || trap cleanup EXIT INT TERM
node_names() { echo "$OUT" | jq -r 'keys[]'; }
node_field() { echo "$OUT" | jq -r --arg n "$1" --arg f "$2" '.[$n][$f]'; }
launch_node() {
n="$1"
for p in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].config_files | keys[]?'); do
mkdir -p "$(dirname "$p")"
echo "$OUT" | jq -r --arg n "$n" --arg p "$p" '.[$n].config_files[$p]' > "$p"
done
for d in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].data_dirs[]?'); do mkdir -p "$d"; done
ENVS=()
while IFS= read -r e; do [ -n "$e" ] && ENVS+=("$e"); done \
< <(echo "$OUT" | jq -r --arg n "$n" '.[$n].env | to_entries[] | "\(.key)=\(.value)"')
ARGV=()
while IFS= read -r a; do ARGV+=("$a"); done \
< <(echo "$OUT" | jq -r --arg n "$n" '.[$n].argv[]')
info "launching $n"
if [ "${#ENVS[@]}" -gt 0 ]; then
env "${ENVS[@]}" "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
else
"$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
fi
echo "$!" > "$RUNDIR/$n.pid"
}
wait_http() {
url="$1"; t="${2:-30}"; i=0
while [ "$i" -lt "$t" ]; do
curl -fsS -o /dev/null --max-time 2 "$url" 2>/dev/null && return 0
i=$((i + 1)); sleep 1
done
return 1
}
# ---- launch masters and wait for quorum -------------------------------------
for n in $(node_names); do
case "$(node_field "$n" role)" in master) launch_node "$n";; esac
done
info "waiting for master quorum across ports: $(mports | tr '\n' ' ')"
QUORUM_OK=0; i=0
while [ "$i" -lt 40 ]; do
LEADERS=0; AGREED_LEADER=""; mismatch=0; reachable=0
for port in $(mports); do
st="$(curl -fsS --max-time 2 "http://127.0.0.1:$port/cluster/status" 2>/dev/null)" || continue
reachable=$((reachable + 1))
[ "$(echo "$st" | jq -r '.IsLeader // false')" = "true" ] && LEADERS=$((LEADERS + 1))
ldr="$(echo "$st" | jq -r '.Leader // ""')"
if [ -n "$ldr" ]; then
if [ -z "$AGREED_LEADER" ]; then AGREED_LEADER="$ldr"; elif [ "$AGREED_LEADER" != "$ldr" ]; then mismatch=1; fi
fi
done
if [ "$reachable" -eq "$NMASTERS" ] && [ "$LEADERS" -eq 1 ] && [ "$mismatch" -eq 0 ] && [ -n "$AGREED_LEADER" ]; then
QUORUM_OK=1; break
fi
i=$((i + 1)); sleep 1
done
if [ "$QUORUM_OK" -eq 1 ]; then
ok "$NMASTERS-master quorum: exactly one leader ($AGREED_LEADER), all agree"
else
bad "master quorum (reachable=$reachable leaders=$LEADERS leader='$AGREED_LEADER')"
fi
# ---- volume -----------------------------------------------------------------
for n in $(node_names); do
case "$(node_field "$n" role)" in volume) launch_node "$n";; esac
done
wait_http "http://127.0.0.1:$VPORT/healthz" 40 && ok "volume /healthz up" || bad "volume /healthz"
ASSIGN_OK=0; i=0
while [ "$i" -lt 20 ]; do
fid="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT1/dir/assign" 2>/dev/null | jq -r '.fid // ""')"
[ -n "$fid" ] && { ASSIGN_OK=1; break; }
i=$((i + 1)); sleep 1
done
[ "$ASSIGN_OK" -eq 1 ] && ok "master assigned fid ($fid) => volume registered" || bad "master /dir/assign (no volume?)"
# ---- filer ------------------------------------------------------------------
for n in $(node_names); do
case "$(node_field "$n" role)" in filer) launch_node "$n";; esac
done
wait_http "http://127.0.0.1:$FPORT/" 30 && ok "filer / up" || bad "filer /"
PAYLOAD="seaweedfs-terraform-smoke-$$"
echo "$PAYLOAD" > "$WORKDIR/hello.txt"
curl -fsS -F "file=@$WORKDIR/hello.txt" "http://127.0.0.1:$FPORT/smoke/hello.txt" >/dev/null 2>&1
got="$(curl -fsS --max-time 5 "http://127.0.0.1:$FPORT/smoke/hello.txt" 2>/dev/null)"
[ "$got" = "$PAYLOAD" ] && ok "filer PUT/GET round-trip" || bad "filer round-trip (got '$got')"
# ---- s3 ---------------------------------------------------------------------
for n in $(node_names); do
case "$(node_field "$n" role)" in s3) launch_node "$n";; esac
done
wait_http "http://127.0.0.1:$SPORT/status" 30 && ok "s3 /status up" || bad "s3 /status"
curl -fsS -X PUT "http://127.0.0.1:$SPORT/smoke-bucket" >/dev/null 2>&1
sleep 1
curl -fsS -X PUT --data-binary "$PAYLOAD" "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" >/dev/null 2>&1
s3got="$(curl -fsS --max-time 5 "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" 2>/dev/null)"
[ "$s3got" = "$PAYLOAD" ] && ok "s3 PUT/GET round-trip" || bad "s3 round-trip (got '$s3got')"
echo
info "RESULTS: $PASS passed, $FAIL failed"
[ "${KEEP:-0}" = "1" ] && info "KEEP=1: cluster left running. Logs in $LOGDIR."
[ "$FAIL" -eq 0 ]
+36
View File
@@ -0,0 +1,36 @@
.PHONY: all build test test-verbose test-short test-debug clean deps tidy
all: build test
# Build the weed binary first
build:
cd ../../weed && go build -o weed .
# Install test dependencies
deps:
go mod download
# Run all tests
test: build deps
go test -timeout 5m ./...
# Run tests with verbose output
test-verbose: build deps
go test -v -timeout 5m ./...
# Skip long-running integration tests
test-short: deps
go test -short -v ./...
# Run tests with debug output from SeaweedFS
test-debug: build deps
go test -v -timeout 5m ./... 2>&1 | tee test.log
# Clean up test artifacts
clean:
rm -f test.log
go clean -testcache
# Update go.sum
tidy:
go mod tidy
+92
View File
@@ -0,0 +1,92 @@
# SeaweedFS NFS Integration Tests
End-to-end tests that boot a real SeaweedFS cluster (`master` + `volume` +
`filer`) plus the experimental `weed nfs` frontend and drive it through the
NFSv3 wire protocol. The tests talk to the server over TCP using
`github.com/willscott/go-nfs-client`, which means they do **not** need a
kernel NFS mount, privileged ports, or any platform-specific tooling.
## Prerequisites
1. Build the `weed` binary:
```bash
cd ../../weed
go build -o weed .
```
2. Go 1.24 or later.
## Running the tests
```bash
# Build weed and run everything
make test
# Verbose output, keeps the subprocess stdout
make test-verbose
# Skip integration tests — useful when iterating on the framework itself
make test-short
# Run a single test
go test -v -run TestNfsBasicReadWrite ./...
```
Every test starts its own cluster on random loopback ports, so runs are
isolated and can execute in parallel.
## Layout
- `framework.go` — launches `weed master`, `weed volume`, `weed filer`, and
`weed nfs` as subprocesses, waits for each to accept TCP, and exposes a
`Mount()` helper that returns an `nfsclient.Target`.
- `basic_test.go` — covers the most common NFS operations:
- Read/write round-trip (`TestNfsBasicReadWrite`)
- Mkdir / ReadDirPlus / RmDir (`TestNfsMkdirAndRmdir`)
- Nested directory + leaf file (`TestNfsNestedDirectories`)
- Rename preserves content (`TestNfsRenamePreservesContent`)
- Overwrite shrinks file size (`TestNfsOverwriteShrinksFile`)
- Large binary file round-trip (`TestNfsLargeFile`)
- Arbitrary binary and empty files (`TestNfsBinaryAndEmptyFiles`)
- Symlink + Readlink (`TestNfsSymlinkRoundTrip`)
- ReadDirPlus ordering sanity (`TestNfsReadDirPlusOrdering`)
- Remove on missing path errors cleanly (`TestNfsRemoveMissingFailsCleanly`)
- FSINFO advertises non-zero limits (`TestNfsFSInfoReturnsSaneLimits`)
- Sequential append writes concatenate (`TestNfsAppendIsSequential`)
- ReadDir after remove (`TestNfsReadDirAfterRemove`)
## Debugging a failing test
Keep the cluster temp dir for inspection:
```go
config := DefaultTestConfig()
config.SkipCleanup = true
```
Enable subprocess stdout/stderr:
```go
config := DefaultTestConfig()
config.EnableDebug = true
```
Or run with `-v`, which flips `EnableDebug` automatically via `testing.Verbose()`.
## Notes
- The NFS server binds to `127.0.0.1` with `-ip.bind=127.0.0.1` and exports
`/nfs_export`. The test framework pre-creates that directory via the
filer's HTTP API before starting the NFS server — the NFS server requires
its export root to exist in the filer's namespace with a real entry, and
the filer's synthetic `/` root does not match the `Name=="/"` check the
NFS server performs during `ensureIndexedEntry`.
- Ports are allocated dynamically. Each test run opens a short-lived
listener on `127.0.0.1:0`, reads back the assigned port, closes the
listener, and hands the port to `weed master/volume/filer/nfs`. There is
a tiny race window between close and reopen that has not been a problem
in practice but is worth remembering if you see a "bind: address already
in use" failure.
- All four `weed` components are started with explicit `-port.grpc=...`
flags. Without them, the default is `-port + 10000`, which overflows
`65535` whenever the HTTP port lands above `55535` — the kernel's
ephemeral port range on macOS routinely does.
+400
View File
@@ -0,0 +1,400 @@
package nfs
import (
"bytes"
"fmt"
"io"
"os"
"path"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
nfsclient "github.com/willscott/go-nfs-client/nfs"
)
// setupFramework is a small helper that boots the cluster for a single test
// and tears everything down on completion. Every test gets a fresh filer +
// volume pair so they cannot step on each other's namespace.
func setupFramework(t *testing.T) *NfsTestFramework {
t.Helper()
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
config := DefaultTestConfig()
config.EnableDebug = testing.Verbose()
fw := NewNfsTestFramework(t, config)
require.NoError(t, fw.Setup(config), "framework setup")
t.Cleanup(fw.Cleanup)
return fw
}
// writeAll writes payload to path on the target in a single Write call. The
// NFS WRITE3 RPC chunks internally, so this exists purely so tests read
// linearly.
func writeAll(t *testing.T, target *nfsclient.Target, remotePath string, payload []byte) {
t.Helper()
file, err := target.OpenFile(remotePath, 0o644)
require.NoError(t, err, "open %s for write", remotePath)
if len(payload) > 0 {
n, err := file.Write(payload)
require.NoError(t, err, "write %s", remotePath)
require.Equal(t, len(payload), n, "short write on %s", remotePath)
}
require.NoError(t, file.Close(), "close %s", remotePath)
}
// readAll opens path on the target and returns the full file contents.
func readAll(t *testing.T, target *nfsclient.Target, remotePath string) []byte {
t.Helper()
file, err := target.Open(remotePath)
require.NoError(t, err, "open %s for read", remotePath)
defer file.Close()
content, err := io.ReadAll(file)
require.NoError(t, err, "read %s", remotePath)
return content
}
// TestNfsBasicReadWrite exercises the most common NFS path: OpenFile + Write
// + Close followed by Open + Read to verify round-trip data integrity.
func TestNfsBasicReadWrite(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
payload := []byte("hello from seaweedfs nfs integration test")
writeAll(t, target, "/hello.txt", payload)
got := readAll(t, target, "/hello.txt")
assert.Equal(t, payload, got, "round-tripped content must match")
info, err := target.Getattr("/hello.txt")
require.NoError(t, err)
assert.Equal(t, int64(len(payload)), int64(info.Filesize))
}
// TestNfsMkdirAndRmdir covers Mkdir, ReadDirPlus, and RmDir. The readdir
// assertion also verifies that the newly-created directory shows up under
// the export root the way a POSIX client would expect.
func TestNfsMkdirAndRmdir(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
_, err = target.Mkdir("/dir1", 0o755)
require.NoError(t, err)
entries, err := target.ReadDirPlus("/")
require.NoError(t, err)
found := false
for _, entry := range entries {
if entry.Name() == "dir1" {
found = true
assert.True(t, entry.IsDir(), "dir1 should be a directory")
}
}
assert.True(t, found, "expected dir1 in readdir listing")
require.NoError(t, target.RmDir("/dir1"))
// After removal, dir1 must be gone from the listing.
entries, err = target.ReadDirPlus("/")
require.NoError(t, err)
for _, entry := range entries {
assert.NotEqual(t, "dir1", entry.Name(), "dir1 should be removed")
}
}
// TestNfsNestedDirectories ensures the server can materialise a deep tree in
// a single Mkdir-per-segment sequence and that reads/writes work at the
// leaves.
func TestNfsNestedDirectories(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
for _, segment := range []string{"/a", "/a/b", "/a/b/c"} {
_, err := target.Mkdir(segment, 0o755)
require.NoError(t, err, "mkdir %s", segment)
}
payload := []byte("deep path content")
writeAll(t, target, "/a/b/c/leaf.txt", payload)
got := readAll(t, target, "/a/b/c/leaf.txt")
assert.Equal(t, payload, got)
require.NoError(t, target.Remove("/a/b/c/leaf.txt"))
require.NoError(t, target.RmDir("/a/b/c"))
require.NoError(t, target.RmDir("/a/b"))
require.NoError(t, target.RmDir("/a"))
}
// TestNfsRenamePreservesContent renames a file and makes sure the content
// at the new path matches what was written at the old one, and that the
// old path disappears. It does not assert on inode identity because pjdfstest
// already covers that and this test intentionally avoids depending on the
// mount-side identity plumbing.
func TestNfsRenamePreservesContent(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
payload := []byte("rename me")
writeAll(t, target, "/src.txt", payload)
require.NoError(t, target.Rename("/src.txt", "/dst.txt"))
_, _, err = target.Lookup("/src.txt")
assert.Error(t, err, "source should be gone after rename")
got := readAll(t, target, "/dst.txt")
assert.Equal(t, payload, got)
require.NoError(t, target.Remove("/dst.txt"))
}
// TestNfsOverwriteShrinksFile rewrites an existing file with shorter content
// and asserts Getattr reports the new (smaller) size. go-nfs-client's
// OpenFile does not pass O_TRUNC, so the test truncates explicitly via
// Setattr(size=0) before the second write — mirroring what `echo >file`
// does on a POSIX client.
func TestNfsOverwriteShrinksFile(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
writeAll(t, target, "/overwrite.txt", []byte("the quick brown fox"))
require.NoError(t, target.Setattr("/overwrite.txt", nfsclient.Sattr3{
Size: nfsclient.SetSize{SetIt: true, Size: 0},
}))
writeAll(t, target, "/overwrite.txt", []byte("short"))
info, err := target.Getattr("/overwrite.txt")
require.NoError(t, err)
assert.Equal(t, int64(len("short")), int64(info.Filesize))
got := readAll(t, target, "/overwrite.txt")
assert.Equal(t, []byte("short"), got)
require.NoError(t, target.Remove("/overwrite.txt"))
}
// TestNfsLargeFile writes a multi-megabyte payload so the write path has to
// cut chunks and flush through the volume server rather than inlining
// content in the filer entry.
func TestNfsLargeFile(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
const size = 3 * 1024 * 1024 // 3 MiB — exceeds the 4 MiB inline cutoff boundary when combined with metadata
payload := make([]byte, size)
for i := range payload {
payload[i] = byte(i % 251) // non-repeating to catch offset bugs
}
writeAll(t, target, "/big.bin", payload)
info, err := target.Getattr("/big.bin")
require.NoError(t, err)
assert.Equal(t, int64(size), int64(info.Filesize))
got := readAll(t, target, "/big.bin")
require.Equal(t, size, len(got))
assert.True(t, bytes.Equal(payload, got), "large file content must round-trip byte-for-byte")
require.NoError(t, target.Remove("/big.bin"))
}
// TestNfsBinaryAndEmptyFiles covers two edge-case payloads the write path
// tends to regress on: arbitrary binary bytes and zero-length files.
func TestNfsBinaryAndEmptyFiles(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
t.Run("AllByteValues", func(t *testing.T) {
payload := make([]byte, 256)
for i := range payload {
payload[i] = byte(i)
}
writeAll(t, target, "/binary.bin", payload)
assert.Equal(t, payload, readAll(t, target, "/binary.bin"))
require.NoError(t, target.Remove("/binary.bin"))
})
t.Run("EmptyFile", func(t *testing.T) {
writeAll(t, target, "/empty.txt", nil)
info, err := target.Getattr("/empty.txt")
require.NoError(t, err)
assert.Equal(t, int64(0), int64(info.Filesize))
require.NoError(t, target.Remove("/empty.txt"))
})
}
// TestNfsSymlinkRoundTrip covers Symlink and Readlink through the nfs server.
// Readlink returns the target path; the server does not auto-traverse it.
func TestNfsSymlinkRoundTrip(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
// Symlink uses a different RPC than open+create, and our server routes it
// through the billy Change interface.
require.NoError(t, target.Symlink("/target.txt", "/link.txt"))
// The underlying target does not need to exist for readlink to succeed.
file, _, err := target.Lookup("/link.txt")
require.NoError(t, err, "lookup symlink")
assert.True(t, file.Mode()&os.ModeSymlink != 0, "expected symlink mode, got %s", file.Mode())
require.NoError(t, target.Remove("/link.txt"))
}
// TestNfsReadDirPlusOrdering creates a handful of files with distinct names
// and ensures ReadDirPlus surfaces every one of them. The server pages
// listings from the filer, so we want to make sure nothing is truncated.
func TestNfsReadDirPlusOrdering(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
_, err = target.Mkdir("/listing", 0o755)
require.NoError(t, err)
names := []string{"alpha.txt", "beta.txt", "gamma.txt", "delta.txt", "epsilon.txt"}
for _, name := range names {
writeAll(t, target, path.Join("/listing", name), []byte(name))
}
entries, err := target.ReadDirPlus("/listing")
require.NoError(t, err)
seen := make(map[string]struct{}, len(entries))
for _, entry := range entries {
if entry.Name() == "." || entry.Name() == ".." {
continue
}
seen[entry.Name()] = struct{}{}
}
for _, name := range names {
_, ok := seen[name]
assert.True(t, ok, "expected %s in directory listing", name)
}
for _, name := range names {
require.NoError(t, target.Remove(path.Join("/listing", name)))
}
require.NoError(t, target.RmDir("/listing"))
}
// TestNfsRemoveMissingFailsCleanly asserts that removing a non-existent path
// surfaces an error instead of silently succeeding. A bug where the server
// returned NFS3_OK on missing entries would hide metadata drift.
func TestNfsRemoveMissingFailsCleanly(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
err = target.Remove("/does_not_exist.txt")
require.Error(t, err, "removing a missing file must error")
// NFS3 surfaces this as NFS3ERR_NOENT; make sure the error text is
// recognisable without locking us into the library's exact wording.
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "noent") ||
strings.Contains(strings.ToLower(err.Error()), "not exist") ||
strings.Contains(strings.ToLower(err.Error()), "no such"),
"unexpected error shape: %v", err)
}
// TestNfsFSInfoReturnsSaneLimits pokes at FSINFO so we catch regressions
// where the server advertises zero read/write limits (which would make
// clients fall back to the 8 KiB floor and slow every test that follows).
func TestNfsFSInfoReturnsSaneLimits(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
info, err := target.FSInfo()
require.NoError(t, err)
require.NotNil(t, info)
assert.Greater(t, info.RTPref, uint32(0), "rtpref must be positive")
assert.Greater(t, info.WTPref, uint32(0), "wtpref must be positive")
}
// TestNfsAppendIsSequential writes two chunks to the same file in separate
// Open cycles and asserts the concatenation is preserved. The second write
// uses O_APPEND (the default Open path in go-nfs-client does not pass
// flags, so we explicitly reopen after writing the first chunk).
func TestNfsAppendIsSequential(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
const prefix = "part1-"
const suffix = "part2"
writeAll(t, target, "/concat.txt", []byte(prefix))
file, err := target.OpenFile("/concat.txt", 0o644)
require.NoError(t, err)
// Seek to end before writing so we append rather than overwrite. go-nfs
// client's File.Seek uses the same offset tracking as Write so this is
// enough to place the second chunk after the first.
_, err = file.Seek(int64(len(prefix)), io.SeekStart)
require.NoError(t, err)
_, err = file.Write([]byte(suffix))
require.NoError(t, err)
require.NoError(t, file.Close())
got := readAll(t, target, "/concat.txt")
assert.Equal(t, prefix+suffix, string(got))
require.NoError(t, target.Remove("/concat.txt"))
}
// Regression: readdir should not emit stale entries after a remove. This is
// the scenario the PR's meta cache invalidation logic was written to fix.
func TestNfsReadDirAfterRemove(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
_, err = target.Mkdir("/churn", 0o755)
require.NoError(t, err)
for i := 0; i < 5; i++ {
writeAll(t, target, path.Join("/churn", fmt.Sprintf("f%d.txt", i)), []byte{byte(i)})
}
// Remove the middle one and re-list.
require.NoError(t, target.Remove("/churn/f2.txt"))
entries, err := target.ReadDirPlus("/churn")
require.NoError(t, err)
for _, entry := range entries {
assert.NotEqual(t, "f2.txt", entry.Name(), "removed file should not reappear in listing")
}
for i := 0; i < 5; i++ {
if i == 2 {
continue
}
require.NoError(t, target.Remove(path.Join("/churn", fmt.Sprintf("f%d.txt", i))))
}
require.NoError(t, target.RmDir("/churn"))
}
+423
View File
@@ -0,0 +1,423 @@
package nfs
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/stretchr/testify/require"
nfsclient "github.com/willscott/go-nfs-client/nfs"
"github.com/willscott/go-nfs-client/nfs/rpc"
)
// NfsTestFramework boots a minimal SeaweedFS cluster (master + volume + filer)
// plus the experimental `weed nfs` frontend and hands out NFSv3 RPC clients
// that talk to it. Everything is driven via subprocesses so the tests exercise
// the same binary an operator would deploy, and no kernel mount is required.
type NfsTestFramework struct {
t *testing.T
tempDir string
dataDir string
masterProcess *os.Process
volumeProcess *os.Process
filerProcess *os.Process
nfsProcess *os.Process
masterAddr string
masterGrpc int
volumeAddr string
volumeGrpc int
filerAddr string
filerGrpc int
nfsAddr string
exportRoot string
weedBinary string
isSetup bool
skipCleanup bool
}
// TestConfig controls how the framework boots the cluster.
type TestConfig struct {
NumVolumes int
EnableDebug bool
SkipCleanup bool // keep temp dir on failure for inspection
// ExportRoot is the filer path the NFS server exports. Defaults to "/"
// so tests can use any path, with a single warning logged by the server.
ExportRoot string
}
// DefaultTestConfig returns the defaults used by most tests. A dedicated
// /nfs_export subtree is used as the NFS export root because the NFS server
// requires the export directory to exist in the filer's namespace and carry
// a non-zero inode — passing "/" would succeed only for filer setups that
// have already backfilled the root inode.
func DefaultTestConfig() *TestConfig {
return &TestConfig{
NumVolumes: 3,
EnableDebug: false,
SkipCleanup: false,
ExportRoot: "/nfs_export",
}
}
// NewNfsTestFramework allocates a framework bound to the current test. Call
// Setup next to actually start the cluster.
func NewNfsTestFramework(t *testing.T, config *TestConfig) *NfsTestFramework {
if config == nil {
config = DefaultTestConfig()
}
tempDir, err := os.MkdirTemp("", "seaweedfs_nfs_test_")
require.NoError(t, err)
// testutil.MustAllocatePorts holds every listener open until the full
// batch has been reserved, which avoids the "close-then-hope" race my
// original per-port helper had. We need seven ports: four HTTP (master,
// volume, filer, nfs) and three gRPC (master, volume, filer — nfs has
// no gRPC endpoint).
ports := testutil.MustAllocatePorts(t, 7)
exportRoot := config.ExportRoot
if exportRoot == "" {
exportRoot = "/"
}
return &NfsTestFramework{
t: t,
tempDir: tempDir,
dataDir: filepath.Join(tempDir, "data"),
masterAddr: fmt.Sprintf("127.0.0.1:%d", ports[0]),
masterGrpc: ports[1],
volumeAddr: fmt.Sprintf("127.0.0.1:%d", ports[2]),
volumeGrpc: ports[3],
filerAddr: fmt.Sprintf("127.0.0.1:%d", ports[4]),
filerGrpc: ports[5],
nfsAddr: fmt.Sprintf("127.0.0.1:%d", ports[6]),
exportRoot: exportRoot,
weedBinary: findWeedBinary(),
isSetup: false,
skipCleanup: config.SkipCleanup,
}
}
// Setup starts the SeaweedFS cluster and the NFS frontend, waiting for each
// component to accept connections before moving on.
func (f *NfsTestFramework) Setup(config *TestConfig) error {
if f.isSetup {
return fmt.Errorf("framework already setup")
}
dirs := []string{
f.dataDir,
filepath.Join(f.dataDir, "master"),
filepath.Join(f.dataDir, "volume"),
}
for _, dir := range dirs {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %v", dir, err)
}
}
if err := f.startMaster(config); err != nil {
return fmt.Errorf("failed to start master: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.masterAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("master not ready at %s", f.masterAddr)
}
if err := f.startVolumeServer(config); err != nil {
return fmt.Errorf("failed to start volume server: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.volumeAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("volume server not ready at %s", f.volumeAddr)
}
if err := f.startFiler(config); err != nil {
return fmt.Errorf("failed to start filer: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.filerAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("filer not ready at %s", f.filerAddr)
}
// Pre-create the export root in the filer's namespace. The NFS server
// expects its export directory to exist with a real inode; uploading a
// placeholder file creates the parent directory implicitly and then
// removing the file leaves the empty directory in place.
if f.exportRoot != "/" {
if err := f.ensureExportRootExists(); err != nil {
return fmt.Errorf("failed to pre-create export root %s: %v", f.exportRoot, err)
}
}
if err := f.startNfsServer(config); err != nil {
return fmt.Errorf("failed to start NFS server: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.nfsAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("NFS server not ready at %s", f.nfsAddr)
}
// Let the NFS server finish wiring up its gRPC subscription to the filer
// before the first client call hits MOUNT/LOOKUP.
time.Sleep(500 * time.Millisecond)
f.isSetup = true
return nil
}
// Cleanup stops all processes. Temp state is preserved if SkipCleanup is set.
func (f *NfsTestFramework) Cleanup() {
processes := []*os.Process{f.nfsProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
for _, proc := range processes {
if proc != nil {
_ = proc.Signal(syscall.SIGTERM)
_, _ = proc.Wait()
}
}
if !f.skipCleanup {
_ = os.RemoveAll(f.tempDir)
}
}
// NfsAddr returns the TCP address the NFS server is listening on.
func (f *NfsTestFramework) NfsAddr() string { return f.nfsAddr }
// FilerAddr returns the TCP address of the filer.
func (f *NfsTestFramework) FilerAddr() string { return f.filerAddr }
// ExportRoot returns the path the NFS server exports.
func (f *NfsTestFramework) ExportRoot() string { return f.exportRoot }
// Mount opens an NFSv3 MOUNT+NFS connection against the running NFS server
// and returns a Target that tests can drive like a mini-VFS. Caller is
// responsible for calling the returned cleanup func to Unmount and close the
// TCP connection.
func (f *NfsTestFramework) Mount() (*nfsclient.Target, func(), error) {
var (
client *rpc.Client
err error
)
// The NFS server's TCP listener may already be accepting connections when
// waitForService returns, but the RPC program registration can trail it
// by a few milliseconds. Retry the dial to absorb that small window.
for attempt := 0; attempt < 20; attempt++ {
client, err = rpc.DialTCP("tcp", f.nfsAddr, false)
if err == nil {
break
}
time.Sleep(25 * time.Millisecond)
}
if err != nil {
return nil, nil, fmt.Errorf("dial NFS: %w", err)
}
// Note: do not set Mount.Addr here. When Addr is non-empty, the go-nfs
// client re-dials via portmapper and concatenates `:111` onto the
// address, which produces "too many colons" for a raw `host:port`
// string. Reusing the existing RPC client avoids that path entirely.
mounter := &nfsclient.Mount{Client: client}
target, err := mounter.Mount(f.exportRoot, rpc.AuthNull)
if err != nil {
client.Close()
return nil, nil, fmt.Errorf("mount %s: %w", f.exportRoot, err)
}
cleanup := func() {
_ = mounter.Unmount()
client.Close()
}
return target, cleanup, nil
}
func (f *NfsTestFramework) startMaster(config *TestConfig) error {
_, masterPort := splitHostPort(f.masterAddr)
args := []string{
"master",
"-ip=127.0.0.1",
fmt.Sprintf("-port=%d", masterPort),
fmt.Sprintf("-port.grpc=%d", f.masterGrpc),
"-mdir=" + filepath.Join(f.dataDir, "master"),
"-raftBootstrap",
"-peers=none",
}
return f.startProcess(&f.masterProcess, config, args)
}
func (f *NfsTestFramework) startVolumeServer(config *TestConfig) error {
_, volumePort := splitHostPort(f.volumeAddr)
// pb.ServerAddress encodes a non-default gRPC port as `host:port.grpc`.
// See weed/pb/server_address.go — the dot, not a colon, is the separator
// between the HTTP port and the gRPC port.
masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
args := []string{
"volume",
"-master=" + masterWithGrpc,
"-ip=127.0.0.1",
fmt.Sprintf("-port=%d", volumePort),
fmt.Sprintf("-port.grpc=%d", f.volumeGrpc),
"-dir=" + filepath.Join(f.dataDir, "volume"),
fmt.Sprintf("-max=%d", config.NumVolumes),
}
return f.startProcess(&f.volumeProcess, config, args)
}
func (f *NfsTestFramework) startFiler(config *TestConfig) error {
_, filerPort := splitHostPort(f.filerAddr)
masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
args := []string{
"filer",
"-master=" + masterWithGrpc,
"-ip=127.0.0.1",
fmt.Sprintf("-port=%d", filerPort),
fmt.Sprintf("-port.grpc=%d", f.filerGrpc),
}
return f.startProcess(&f.filerProcess, config, args)
}
func (f *NfsTestFramework) startNfsServer(config *TestConfig) error {
_, nfsPort := splitHostPort(f.nfsAddr)
// `host:port.grpc` encoding — see pb/server_address.go.
filerWithGrpc := fmt.Sprintf("%s.%d", f.filerAddr, f.filerGrpc)
args := []string{
"nfs",
"-filer=" + filerWithGrpc,
"-ip.bind=127.0.0.1",
fmt.Sprintf("-port=%d", nfsPort),
"-filer.path=" + f.exportRoot,
}
return f.startProcess(&f.nfsProcess, config, args)
}
func (f *NfsTestFramework) startProcess(target **os.Process, config *TestConfig, args []string) error {
cmd := exec.Command(f.weedBinary, args...)
cmd.Dir = f.tempDir
if config.EnableDebug {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
}
if err := cmd.Start(); err != nil {
return err
}
*target = cmd.Process
return nil
}
// portFromAddr returns just the port number from a `host:port` string.
// testutil.WaitForPort takes an int port, not a full address.
func portFromAddr(addr string) int {
_, port := splitHostPort(addr)
return port
}
// ensureExportRootExists posts a placeholder file to f.exportRoot via the
// filer's HTTP API, then deletes it. That roundtrip implicitly creates the
// target directory so the NFS server has something to mount. We bypass
// weed/pb here because the HTTP client is simpler and needs no gRPC stubs.
func (f *NfsTestFramework) ensureExportRootExists() error {
exportRoot := strings.TrimRight(f.exportRoot, "/")
if exportRoot == "" {
return nil
}
placeholder := exportRoot + "/.nfs_test_init"
filerURL := "http://" + f.filerAddr + placeholder
var body bytes.Buffer
writer := multipart.NewWriter(&body)
part, err := writer.CreateFormFile("file", ".nfs_test_init")
if err != nil {
return err
}
if _, err := io.WriteString(part, ""); err != nil {
return err
}
if err := writer.Close(); err != nil {
return err
}
httpClient := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest(http.MethodPost, filerURL, &body)
if err != nil {
return err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := httpClient.Do(req)
if err != nil {
return err
}
_, _ = io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("filer POST %s returned status %d", filerURL, resp.StatusCode)
}
// Delete the placeholder; the directory stays behind.
deleteReq, err := http.NewRequest(http.MethodDelete, filerURL, nil)
if err != nil {
return err
}
deleteResp, err := httpClient.Do(deleteReq)
if err != nil {
return err
}
_, _ = io.Copy(io.Discard, deleteResp.Body)
deleteResp.Body.Close()
if deleteResp.StatusCode/100 != 2 && deleteResp.StatusCode != http.StatusNotFound {
return fmt.Errorf("filer DELETE %s returned status %d", filerURL, deleteResp.StatusCode)
}
return nil
}
func splitHostPort(addr string) (string, int) {
host, portStr, err := net.SplitHostPort(addr)
if err != nil {
return "", 0
}
var port int
_, _ = fmt.Sscanf(portStr, "%d", &port)
return host, port
}
// findWeedBinary locates the weed binary, preferring the local build in the
// checkout so tests run against the code under review rather than whatever is
// on $PATH.
func findWeedBinary() string {
if _, thisFile, _, ok := runtime.Caller(0); ok {
thisDir := filepath.Dir(thisFile)
candidates := []string{
filepath.Join(thisDir, "../../weed/weed"),
filepath.Join(thisDir, "../weed/weed"),
}
for _, candidate := range candidates {
if _, err := os.Stat(candidate); err == nil {
abs, _ := filepath.Abs(candidate)
return abs
}
}
}
cwd, _ := os.Getwd()
candidates := []string{
filepath.Join(cwd, "../../weed/weed"),
filepath.Join(cwd, "../weed/weed"),
filepath.Join(cwd, "./weed"),
}
for _, candidate := range candidates {
if _, err := os.Stat(candidate); err == nil {
abs, _ := filepath.Abs(candidate)
return abs
}
}
if path, err := exec.LookPath("weed"); err == nil {
return path
}
return "weed"
}
+21
View File
@@ -0,0 +1,21 @@
module seaweedfs-nfs-tests
go 1.25.0
// test/testutil lives inside the main seaweedfs module; pull it in via a
// local replace so this integration suite can reuse the shared port
// allocator and readiness helpers instead of reinventing them.
replace github.com/seaweedfs/seaweedfs => ../..
require (
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
github.com/stretchr/testify v1.11.1
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
)
require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+14
View File
@@ -0,0 +1,14 @@
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+193
View File
@@ -0,0 +1,193 @@
//go:build linux
package nfs
// End-to-end mount tests that drive the real Linux NFS client (mount.nfs +
// in-tree kernel) against a running `weed nfs` subprocess. These exist to
// catch regressions that the existing framework can't see, because the
// framework drives the server with willscott/go-nfs-client — the same RPC
// library the server uses internally — so any bug shared between the two
// (XDR layout, version dispatch, RPC framing) round-trips invisibly.
//
// Two real bugs hit recently were exactly that shape:
// 1. NFSv4 mis-routed to the v3 SETATTR handler (#9262). The client
// library never sends NFSv4, so the test suite never noticed; the
// Linux kernel mount path did notice, with EIO.
// 2. UDP MOUNT v3 missing. Only TCP MOUNT was advertised; the kernel
// defaults mountproto=udp in many setups, so the in-tree client
// surfaced EPROTONOSUPPORT during MOUNT setup.
//
// These tests mount over the actual loopback interface using mount.nfs and
// shell out to /bin/mount and /bin/umount. They require root (mount(2) is
// privileged) and Linux (the in-tree NFS client is what's being exercised);
// they t.Skip cleanly when either prerequisite is missing.
//
// Run locally with:
//
// cd test/nfs
// sudo go test -v -run TestKernelMount ./...
//
// CI runs them via .github/workflows/nfs-tests.yml after installing
// nfs-common (mount.nfs + helpers).
import (
"errors"
"fmt"
"net"
"os"
"os/exec"
"strings"
"testing"
)
// kernelMountSkipIfUnsupported skips the test when the host can't run a
// real NFS mount. The combined check belongs in one place so the three
// kernel-mount tests stay focused on what they're actually verifying.
func kernelMountSkipIfUnsupported(t *testing.T) {
t.Helper()
if os.Geteuid() != 0 {
t.Skip("kernel mount test requires root; mount(2) is privileged")
}
if _, err := exec.LookPath("mount.nfs"); err != nil {
t.Skipf("mount.nfs not installed: %v (CI installs the nfs-common package)", err)
}
}
// kernelMount runs /bin/mount with the given options against the framework's
// running NFS server, returns the mountpoint and an unmount closure. We pass
// explicit port=/mountport= options so the kernel never queries portmap.
// That keeps the harness honest about what it's testing — the NFS / MOUNT
// wire protocol — and avoids colliding with a system rpcbind on shared CI
// runners (port 111 is privileged and frequently in use already).
func kernelMount(t *testing.T, fw *NfsTestFramework, optsTemplate string) (string, func()) {
t.Helper()
host, portStr, err := net.SplitHostPort(fw.NfsAddr())
if err != nil {
t.Fatalf("split nfs addr %q: %v", fw.NfsAddr(), err)
}
mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-")
if err != nil {
t.Fatalf("mkdtemp: %v", err)
}
opts := strings.ReplaceAll(optsTemplate, "{port}", portStr)
target := fmt.Sprintf("%s:%s", host, fw.ExportRoot())
cmd := exec.Command("mount", "-t", "nfs", "-o", opts, target, mountpoint)
if out, err := cmd.CombinedOutput(); err != nil {
_ = os.RemoveAll(mountpoint)
t.Fatalf("mount %s -o %s failed: %v\nmount output:\n%s", target, opts, err, out)
}
teardown := func() {
// -f to bail out faster if the server's already gone.
_ = exec.Command("umount", "-f", mountpoint).Run()
_ = os.RemoveAll(mountpoint)
}
return mountpoint, teardown
}
func newKernelMountFramework(t *testing.T) *NfsTestFramework {
t.Helper()
cfg := DefaultTestConfig()
fw := NewNfsTestFramework(t, cfg)
if err := fw.Setup(cfg); err != nil {
fw.Cleanup()
t.Fatalf("framework setup: %v", err)
}
t.Cleanup(fw.Cleanup)
return fw
}
// TestKernelMountV3TCP exercises the most common mount form: NFSv3 + MOUNT
// v3, both over TCP. This is what the existing go-nfs-client tests cover at
// the protocol layer, but running it through mount.nfs and the kernel
// confirms that the wire format we emit decodes cleanly under a different
// XDR/RPC parser.
func TestKernelMountV3TCP(t *testing.T) {
kernelMountSkipIfUnsupported(t)
fw := newKernelMountFramework(t)
mountpoint, undo := kernelMount(t, fw,
"nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=tcp")
defer undo()
if _, err := os.Stat(mountpoint); err != nil {
t.Errorf("stat mountpoint: %v", err)
}
if _, err := os.ReadDir(mountpoint); err != nil {
t.Errorf("readdir mountpoint: %v", err)
}
}
// TestKernelMountV3MountProtoUDP is the regression test for the UDP MOUNT
// v3 responder. mountproto=udp forces the kernel to call MOUNT over UDP
// only; before the responder existed the kernel hit nothing (MOUNT was
// advertised TCP-only) and surfaced EPROTONOSUPPORT during mount setup.
func TestKernelMountV3MountProtoUDP(t *testing.T) {
kernelMountSkipIfUnsupported(t)
fw := newKernelMountFramework(t)
mountpoint, undo := kernelMount(t, fw,
"nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=udp")
defer undo()
if _, err := os.Stat(mountpoint); err != nil {
t.Errorf("stat mountpoint: %v", err)
}
}
// TestKernelMountV4RejectsCleanly is the regression test for the NFSv4
// PROG_MISMATCH path (#9262). The server only speaks NFSv3, but the
// previous behaviour was to mis-route v4 COMPOUND to the v3 SETATTR
// handler and write garbage; the kernel surfaced EIO instead of a
// version-mismatch error and (depending on distro) didn't fall back to
// v3. The version filter now answers PROG_MISMATCH so the kernel sees
// "v4 not supported" cleanly.
//
// The test asserts:
// 1. mount.nfs exits non-zero (no silent success against a v3 server);
// 2. the failure message mentions protocol/version/io, which is what the
// kernel surfaces when it gets PROG_MISMATCH instead of garbage. A
// pre-fix server returns "mount system call failed" with no further
// context, so a regression collapses the assertion onto that branch.
func TestKernelMountV4RejectsCleanly(t *testing.T) {
kernelMountSkipIfUnsupported(t)
fw := newKernelMountFramework(t)
host, portStr, err := net.SplitHostPort(fw.NfsAddr())
if err != nil {
t.Fatalf("split nfs addr: %v", err)
}
mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-v4-")
if err != nil {
t.Fatalf("mkdtemp: %v", err)
}
defer os.RemoveAll(mountpoint)
target := fmt.Sprintf("%s:%s", host, fw.ExportRoot())
cmd := exec.Command("mount", "-t", "nfs", "-o",
fmt.Sprintf("vers=4,port=%s", portStr),
target, mountpoint)
out, err := cmd.CombinedOutput()
defer exec.Command("umount", "-f", mountpoint).Run()
if err == nil {
t.Fatalf("v4 mount unexpectedly succeeded against v3-only server\nmount output:\n%s", out)
}
// Don't pin the exact error string — different distros print slightly
// different things — but require some hint that the kernel saw a
// protocol-level failure rather than a generic "mount system call
// failed". Without the version filter, mount.nfs prints the latter
// alone; with it, the former.
lower := strings.ToLower(string(out))
if !strings.Contains(lower, "protocol") &&
!strings.Contains(lower, "version") &&
!strings.Contains(lower, "i/o") {
t.Errorf("v4 mount failure didn't mention protocol/version/io; output:\n%s", out)
}
// Also require a non-zero exit so a future change that makes mount(2)
// silently succeed (e.g. by relaxing the version filter) shows up
// here even if the message phrasing changes.
var ee *exec.ExitError
if !errors.As(err, &ee) {
t.Errorf("expected mount to exit non-zero with ExitError, got %v", err)
}
}
+16 -25
View File
@@ -30,22 +30,21 @@ type VolumeServer struct {
address string
baseDir string
mu sync.Mutex
receivedFiles map[string]uint64
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
deleteRequests []*volume_server_pb.VolumeDeleteRequest
markReadonlyCalls int
markWritableCalls int
readFileStatusCalls int
vacuumGarbageRatio float64
vacuumCommitReadOnly bool
vacuumCheckCalls int
vacuumCompactCalls int
vacuumCommitCalls int
vacuumCleanupCalls int
volumeCopyCalls int
volumeMountCalls int
tailReceiverCalls int
mu sync.Mutex
receivedFiles map[string]uint64
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
deleteRequests []*volume_server_pb.VolumeDeleteRequest
markReadonlyCalls int
markWritableCalls int
readFileStatusCalls int
vacuumGarbageRatio float64
vacuumCheckCalls int
vacuumCompactCalls int
vacuumCommitCalls int
vacuumCleanupCalls int
volumeCopyCalls int
volumeMountCalls int
tailReceiverCalls int
}
// NewVolumeServer starts a test volume server using the provided base directory.
@@ -116,13 +115,6 @@ func (v *VolumeServer) SetVacuumGarbageRatio(ratio float64) {
v.vacuumGarbageRatio = ratio
}
// SetVacuumCommitReadOnly sets the IsReadOnly value returned by VacuumVolumeCommit.
func (v *VolumeServer) SetVacuumCommitReadOnly(readOnly bool) {
v.mu.Lock()
defer v.mu.Unlock()
v.vacuumCommitReadOnly = readOnly
}
// VacuumStats returns the vacuum RPC call counts.
func (v *VolumeServer) VacuumStats() (check, compact, commit, cleanup int) {
v.mu.Lock()
@@ -436,9 +428,8 @@ func (v *VolumeServer) VacuumVolumeCompact(req *volume_server_pb.VacuumVolumeCom
func (v *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_server_pb.VacuumVolumeCommitRequest) (*volume_server_pb.VacuumVolumeCommitResponse, error) {
v.mu.Lock()
v.vacuumCommitCalls++
readOnly := v.vacuumCommitReadOnly
v.mu.Unlock()
return &volume_server_pb.VacuumVolumeCommitResponse{IsReadOnly: readOnly}, nil
return &volume_server_pb.VacuumVolumeCommitResponse{}, nil
}
func (v *VolumeServer) VacuumVolumeCleanup(ctx context.Context, req *volume_server_pb.VacuumVolumeCleanupRequest) (*volume_server_pb.VacuumVolumeCleanupResponse, error) {
+18 -63
View File
@@ -15,24 +15,6 @@ import (
"google.golang.org/grpc/credentials/insecure"
)
func vacuumJobSpec(volumeID uint32, server string) *plugin_pb.JobSpec {
return &plugin_pb.JobSpec{
JobId: fmt.Sprintf("vacuum-job-%d", volumeID),
JobType: "vacuum",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
},
"server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: server},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "vac-test"},
},
},
}
}
func TestVacuumExecutionIntegration(t *testing.T) {
volumeID := uint32(202)
@@ -49,10 +31,26 @@ func TestVacuumExecutionIntegration(t *testing.T) {
source := pluginworkers.NewVolumeServer(t, "")
source.SetVacuumGarbageRatio(0.6)
job := &plugin_pb.JobSpec{
JobId: fmt.Sprintf("vacuum-job-%d", volumeID),
JobType: "vacuum",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
},
"server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source.Address()},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "vac-test"},
},
},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.Success)
@@ -61,48 +59,5 @@ func TestVacuumExecutionIntegration(t *testing.T) {
require.GreaterOrEqual(t, checkCalls, 2)
require.GreaterOrEqual(t, compactCalls, 1)
require.GreaterOrEqual(t, commitCalls, 1)
// Cleanup is only invoked when Phase 1 (Compact) fails to roll back
// the .cpd/.cpx/.cpldb temp files; on the success path Commit
// consumes them (rename .cpd → .dat, .cpx → .idx, .cpldb → .ldb via
// the leveldb needle map) so no Cleanup call is needed. Matches
// topology.vacuumOneVolumeId which only calls batchVacuumVolumeCleanup
// on the Compact-failure branch.
require.Equal(t, 0, cleanupCalls)
// Phase 3 marks each replica writable so master returns it to the
// writables layout. See upstream seaweedfs#9685.
require.GreaterOrEqual(t, source.MarkWritableCount(), 1)
}
// A replica that commits still read-only (operator-set, EIO-quarantined,
// disk-space-low) must not be force-marked writable: master built-in vacuum
// skips it via SetVolumeAvailable, and it recovers on its own ReadOnly=false
// heartbeat.
func TestVacuumExecutionSkipsMarkWritableWhenReadOnly(t *testing.T) {
volumeID := uint32(203)
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
handler := vacuum.NewVacuumHandler(dialOption, 1)
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
WorkerOptions: pluginworker.WorkerOptions{
GrpcDialOption: dialOption,
},
Handlers: []pluginworker.JobHandler{handler},
})
harness.WaitForJobType("vacuum")
source := pluginworkers.NewVolumeServer(t, "")
source.SetVacuumGarbageRatio(0.6)
source.SetVacuumCommitReadOnly(true)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.Success)
_, _, commitCalls, _ := source.VacuumStats()
require.GreaterOrEqual(t, commitCalls, 1)
require.Equal(t, 0, source.MarkWritableCount())
require.GreaterOrEqual(t, cleanupCalls, 1)
}
-107
View File
@@ -1,107 +0,0 @@
{
"realm": "seaweedfs-test",
"enabled": true,
"roles": {
"realm": [
{ "name": "s3-admin" },
{ "name": "s3-read-only" },
{ "name": "s3-write-only" },
{ "name": "s3-read-write" }
]
},
"clients": [
{
"clientId": "seaweedfs-s3",
"enabled": true,
"secret": "seaweedfs-s3-secret",
"publicClient": false,
"serviceAccountsEnabled": true,
"standardFlowEnabled": true,
"directAccessGrantsEnabled": true,
"fullScopeAllowed": true,
"redirectUris": ["*"],
"webOrigins": ["*"],
"protocol": "openid-connect",
"protocolMappers": [
{
"name": "realm-roles",
"protocol": "openid-connect",
"protocolMapper": "oidc-usermodel-realm-role-mapper",
"config": {
"claim.name": "roles",
"jsonType.label": "String",
"multivalued": "true",
"usermodel.realmRoleMapping.rolePrefix": "",
"access.token.claim": "true",
"id.token.claim": "true",
"userinfo.token.claim": "true"
}
},
{
"name": "audience-mapper",
"protocol": "openid-connect",
"protocolMapper": "oidc-audience-mapper",
"config": {
"included.client.audience": "seaweedfs-s3",
"id.token.claim": "false",
"access.token.claim": "true"
}
}
]
}
],
"users": [
{
"username": "admin-user",
"enabled": true,
"email": "admin-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-admin"],
"credentials": [
{ "type": "password", "value": "adminuser123", "temporary": false }
]
},
{
"username": "read-user",
"enabled": true,
"email": "read-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-read-only"],
"credentials": [
{ "type": "password", "value": "readuser123", "temporary": false }
]
},
{
"username": "write-user",
"enabled": true,
"email": "write-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-read-write"],
"credentials": [
{ "type": "password", "value": "writeuser123", "temporary": false }
]
},
{
"username": "write-only-user",
"enabled": true,
"email": "write-only-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-write-only"],
"credentials": [
{ "type": "password", "value": "writeonlyuser123", "temporary": false }
]
}
]
}
+8 -53
View File
@@ -7,7 +7,6 @@ import (
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
@@ -25,7 +24,6 @@ type MultiVolumeCluster struct {
logsDir string
keepLogs bool
volumeServerCount int
disksPerServer int
masterPort int
masterGrpcPort int
@@ -33,7 +31,6 @@ type MultiVolumeCluster struct {
volumePorts []int
volumeGrpcPorts []int
volumePubPorts []int
volumeDiskDirs [][]string // per server: the data directories (one DiskLocation each)
masterCmd *exec.Cmd
volumeCmds []*exec.Cmd
@@ -41,25 +38,13 @@ type MultiVolumeCluster struct {
cleanupOnce sync.Once
}
// StartMultiVolumeCluster starts a cluster with serverCount volume servers, one
// data directory (DiskLocation) each.
// StartMultiVolumeCluster starts a cluster with a specified number of volume servers
func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount int) *MultiVolumeCluster {
return StartMultiVolumeClusterWithDisks(t, profile, serverCount, 1)
}
// StartMultiVolumeClusterWithDisks starts serverCount volume servers, each with
// disksPerServer data directories passed to -dir as a comma list so every
// directory becomes its own DiskLocation. Lets tests exercise per-disk EC
// layouts (e.g. shards on one disk, a stub/index on a sibling disk).
func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serverCount, disksPerServer int) *MultiVolumeCluster {
t.Helper()
if serverCount < 1 {
t.Fatalf("serverCount must be at least 1, got %d", serverCount)
}
if disksPerServer < 1 {
t.Fatalf("disksPerServer must be at least 1, got %d", disksPerServer)
}
weedBinary, err := FindOrBuildWeedBinary()
if err != nil {
@@ -75,21 +60,10 @@ func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serv
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
// Create directories for master and all volume servers. With one disk the
// layout stays at baseDir/volumeN; multi-disk servers get baseDir/volumeN/diskD.
// Create directories for master and all volume servers
dirs := []string{configDir, logsDir, masterDataDir}
volumeDiskDirs := make([][]string, serverCount)
for i := 0; i < serverCount; i++ {
serverDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
volumeDiskDirs[i] = make([]string, disksPerServer)
for d := 0; d < disksPerServer; d++ {
if disksPerServer == 1 {
volumeDiskDirs[i][d] = serverDir
} else {
volumeDiskDirs[i][d] = filepath.Join(serverDir, fmt.Sprintf("disk%d", d))
}
dirs = append(dirs, volumeDiskDirs[i][d])
}
dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
}
for _, dir := range dirs {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
@@ -124,13 +98,11 @@ func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serv
logsDir: logsDir,
keepLogs: keepLogs,
volumeServerCount: serverCount,
disksPerServer: disksPerServer,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
volumePorts: make([]int, serverCount),
volumeGrpcPorts: make([]int, serverCount),
volumePubPorts: make([]int, serverCount),
volumeDiskDirs: volumeDiskDirs,
volumeCmds: make([]*exec.Cmd, serverCount),
}
@@ -161,7 +133,8 @@ func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serv
// Start all volume servers
for i := 0; i < serverCount; i++ {
if err = c.startVolume(i, c.volumeDiskDirs[i]); err != nil {
volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
if err = c.startVolume(i, volumeDataDir); err != nil {
// Log current server's log for debugging startup failures
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
@@ -234,17 +207,13 @@ func (c *MultiVolumeCluster) startMaster(dataDir string) error {
return c.masterCmd.Start()
}
func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
func (c *MultiVolumeCluster) startVolume(index int, dataDir string) error {
logName := fmt.Sprintf("volume%d.log", index)
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
if err != nil {
return err
}
maxPerDir := make([]string, len(dataDirs))
for i := range dataDirs {
maxPerDir[i] = "16"
}
args := []string{
"-config_dir=" + c.configDir,
"volume",
@@ -252,8 +221,8 @@ func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
"-port=" + strconv.Itoa(c.volumePorts[index]),
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
"-dir=" + strings.Join(dataDirs, ","),
"-max=" + strings.Join(maxPerDir, ","),
"-dir=" + dataDir,
"-max=16",
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
"-readMode=" + c.profile.ReadMode,
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
@@ -332,17 +301,3 @@ func (c *MultiVolumeCluster) VolumePublicURL(index int) string {
func (c *MultiVolumeCluster) BaseDir() string {
return c.baseDir
}
// VolumeDiskDir returns the data directory backing disk diskIndex on volume
// server serverIndex — i.e. the directory for that DiskLocation, for file
// inspection in tests.
func (c *MultiVolumeCluster) VolumeDiskDir(serverIndex, diskIndex int) string {
if serverIndex < 0 || serverIndex >= len(c.volumeDiskDirs) {
return ""
}
disks := c.volumeDiskDirs[serverIndex]
if diskIndex < 0 || diskIndex >= len(disks) {
return ""
}
return disks[diskIndex]
}
+1 -1
View File
@@ -301,7 +301,7 @@ func (r *runner) fetchVolumeList(ctx context.Context) (*master_pb.VolumeListResp
}
var response *master_pb.VolumeListResponse
err := pb.WithMasterClient(context.Background(), false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
err := pb.WithMasterClient(false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
callCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
defer cancel()
+1 -1
View File
@@ -1778,7 +1778,7 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col
shards := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)
data := collectionMap[collection]
data.PhysicalSize += int64(shards.TotalSize())
data.LogicalSize += int64(shards.MinusParityShards(erasure_coding.DataShardsCount).TotalSize())
data.LogicalSize += int64(shards.MinusParityShards().TotalSize())
collectionMap[collection] = data
// fileCount is volume-wide (same .ecx on every shard
-1
View File
@@ -246,7 +246,6 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
filesApi.HandleFunc("/download", h.fileBrowserHandlers.DownloadFile).Methods(http.MethodGet)
filesApi.HandleFunc("/view", h.fileBrowserHandlers.ViewFile).Methods(http.MethodGet)
filesApi.HandleFunc("/properties", h.fileBrowserHandlers.GetFileProperties).Methods(http.MethodGet)
filesApi.HandleFunc("/metadata", h.fileBrowserHandlers.ExportMetadata).Methods(http.MethodGet)
volumeApi := api.PathPrefix("/volumes").Subrouter()
volumeApi.Handle("/{id}/{server}/vacuum", wrapWrite(h.clusterHandlers.VacuumVolume)).Methods(http.MethodPost)
+30 -22
View File
@@ -9,10 +9,8 @@ import (
"net/http"
"path"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
@@ -21,8 +19,13 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
)
// Admin upload chunk size, matching s3api so files split into the same fid-sized pieces.
const adminUploadChunkSize = 8 * 1024 * 1024
// Admin file browser upload chunk sizing — kept in sync with the values
// s3api uses so files end up split into the same fid-sized pieces the rest of
// the cluster expects.
const (
adminUploadChunkSize = 8 * 1024 * 1024
adminUploadSmallFileLimit = 256 * 1024
)
// File browser handlers backed by the filer gRPC service. They bypass the
// filer's HTTP listener so the UI keeps working when the filer is started
@@ -64,7 +67,7 @@ func (h *FileBrowserHandlers) fetchFileContentGrpc(ctx context.Context, filePath
// response writer receives the canonical attachment headers and the raw
// bytes; this replaces the HTTP-to-filer proxy that used to run in
// DownloadFile.
func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter, inline bool) error {
func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter) error {
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
return err
@@ -81,28 +84,27 @@ func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath str
size := int64(filer.FileSize(entry))
fileName := path.Base(cleanFilePath)
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
// Resolve mime like the viewer does so Content-Type and the inline check agree.
contentType := dash.ResolveEntryMime(entry)
// Only inline images and PDFs; serve the rest as attachments so a hostile upload
// (HTML, SVG) can't run as same-origin script. nosniff locks the declared type.
disposition := "attachment"
if inline && (strings.HasPrefix(contentType, "image/") || contentType == "application/pdf") {
disposition = "inline"
contentType := ""
if entry.Attributes != nil {
contentType = entry.Attributes.Mime
}
if contentType == "" {
contentType = "application/octet-stream"
}
w.Header().Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": fileName}))
w.Header().Set("Content-Type", contentType)
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
w.WriteHeader(http.StatusOK)
return h.streamEntryContent(ctx, entry, size, w)
}
// uploadFileGrpc streams the upload to volumes in 8 MiB chunks via the shared
// chunked-upload helper, then registers the entry over the filer gRPC service.
// Content always lands in volumes, never inlined on the entry.
// uploadFileGrpc streams the upload to volume servers in 8 MiB chunks via the
// shared chunked-upload helper, then registers the assembled entry through the
// filer gRPC service. Bytes never enter the admin process's heap as a whole —
// each chunk is sized to adminUploadChunkSize. Small files (< 256 KiB) are
// stored inline on the entry, matching the S3 server's behaviour.
func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath string, fileName string, mimeType string, reader io.Reader) error {
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
@@ -150,9 +152,11 @@ func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath strin
}
chunkResult, err := operation.UploadReaderInChunks(ctx, reader, &operation.ChunkedUploadOption{
ChunkSize: adminUploadChunkSize,
MimeType: mimeType,
AssignFunc: assignFunc,
ChunkSize: adminUploadChunkSize,
SmallFileLimit: adminUploadSmallFileLimit,
SaveSmallInline: true,
MimeType: mimeType,
AssignFunc: assignFunc,
})
if err != nil {
// Partial chunks come back even on error so we can clean them up rather
@@ -174,7 +178,11 @@ func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath strin
Mime: mimeType,
},
}
entry.Chunks = chunkResult.FileChunks
if len(chunkResult.SmallContent) > 0 {
entry.Content = chunkResult.SmallContent
} else {
entry.Chunks = chunkResult.FileChunks
}
err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
_, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+2 -95
View File
@@ -1,12 +1,8 @@
package handlers
import (
"bufio"
"compress/gzip"
"context"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
"os"
@@ -19,12 +15,10 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/http/client"
"google.golang.org/protobuf/proto"
)
type FileBrowserHandlers struct {
@@ -397,9 +391,8 @@ func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Reques
writeJSONError(w, http.StatusBadRequest, "File path is required")
return
}
inline := r.URL.Query().Get("inline") == "true"
tracker := &responseWriteTracker{ResponseWriter: w}
if err := h.downloadFileGrpc(r.Context(), filePath, tracker, inline); err != nil {
if err := h.downloadFileGrpc(r.Context(), filePath, tracker); err != nil {
// Once bytes have been written we can't switch to a JSON error body
// without corrupting the partial response — log and stop. Before any
// write the response is still uncommitted, so a 502 with details is
@@ -635,93 +628,6 @@ func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.R
writeJSON(w, http.StatusOK, properties)
}
// ExportMetadata streams a file or folder's metadata as a gzipped, length-prefixed
// FullEntry stream — the format weed shell fs.meta.load reads. Directories are
// walked recursively via the filer BFS metadata stream.
func (h *FileBrowserHandlers) ExportMetadata(w http.ResponseWriter, r *http.Request) {
filePath := r.URL.Query().Get("path")
if filePath == "" {
writeJSONError(w, http.StatusBadRequest, "File path is required")
return
}
cleanPath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
writeJSONError(w, http.StatusBadRequest, err.Error())
return
}
tracker := &responseWriteTracker{ResponseWriter: w}
err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
stream, err := client.TraverseBfsMetadata(r.Context(), &filer_pb.TraverseBfsMetadataRequest{
Directory: cleanPath,
ExcludedPrefixes: []string{filer.SystemLogDir},
})
if err != nil {
return err
}
// Read the first entry before sending headers so a bad path returns a clean error.
first, err := stream.Recv()
if err != nil {
return err
}
downloadName := path.Base(cleanPath)
if downloadName == "/" || downloadName == "." || downloadName == "" {
downloadName = "root"
}
tracker.Header().Set("Content-Type", "application/gzip")
tracker.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": downloadName + ".meta.gz"}))
tracker.WriteHeader(http.StatusOK)
bw := bufio.NewWriter(tracker)
gw := gzip.NewWriter(bw)
sizeBuf := make([]byte, 4)
writeEntry := func(resp *filer_pb.TraverseBfsMetadataResponse) error {
b, err := proto.Marshal(&filer_pb.FullEntry{Dir: resp.Directory, Entry: resp.Entry})
if err != nil {
return err
}
util.Uint32toBytes(sizeBuf, uint32(len(b)))
if _, err := gw.Write(sizeBuf); err != nil {
return err
}
_, err = gw.Write(b)
return err
}
if err := writeEntry(first); err != nil {
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
return recvErr
}
if err := writeEntry(resp); err != nil {
return err
}
}
if err := gw.Close(); err != nil {
return err
}
return bw.Flush()
})
if err != nil {
if tracker.committed {
glog.Errorf("Error exporting metadata for %s: %v", cleanPath, err)
return
}
writeJSONError(w, http.StatusInternalServerError, "Failed to export metadata: "+err.Error())
}
}
// Helper function to format bytes
func (h *FileBrowserHandlers) formatBytes(bytes int64) string {
const unit = 1024
@@ -779,3 +685,4 @@ func min(a, b int64) int64 {
}
return b
}
+2 -2
View File
@@ -1788,7 +1788,7 @@ function createFileViewerContent(file, content) {
if (file.mime.startsWith('image/')) {
return `
<div class="text-center">
<img src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path) + '&inline=true')}"
<img src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
class="img-fluid" alt="${file.name}" style="max-height: 500px;">
</div>
`;
@@ -1806,7 +1806,7 @@ function createFileViewerContent(file, content) {
} else if (file.mime === 'application/pdf') {
return `
<div class="text-center">
<embed src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path) + '&inline=true')}"
<embed src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
type="application/pdf" width="100%" height="500px">
</div>
`;
+1 -12
View File
@@ -198,9 +198,6 @@ templ FileBrowser(data dash.FileBrowserData) {
<button type="button" class="btn btn-outline-secondary btn-sm" title="Properties" data-action="properties" data-path={ entry.FullPath }>
<i class="fas fa-info-circle"></i>
</button>
<button type="button" class="btn btn-outline-success btn-sm" title="Export metadata" data-action="export-metadata" data-path={ entry.FullPath }>
<i class="fas fa-file-export"></i>
</button>
<button type="button" class="btn btn-outline-danger btn-sm" title="Delete" data-action="delete" data-path={ entry.FullPath }>
<i class="fas fa-trash"></i>
</button>
@@ -357,7 +354,7 @@ templ FileBrowser(data dash.FileBrowserData) {
}
});
// Handle file browser action buttons (download, view, properties, export-metadata, delete)
// Handle file browser action buttons (download, view, properties, delete)
document.addEventListener('click', function(e) {
const button = e.target.closest('[data-action]');
if (!button) return;
@@ -377,9 +374,6 @@ templ FileBrowser(data dash.FileBrowserData) {
case 'properties':
showFileProperties(path);
break;
case 'export-metadata':
exportMetadata(path);
break;
case 'delete':
const fileName = path.split('/').pop();
showDeleteConfirm(fileName, function() {
@@ -406,11 +400,6 @@ templ FileBrowser(data dash.FileBrowserData) {
window.open((window.__BASE_PATH__ || '') + '/api/files/view?path=' + encodeURIComponent(path), '_blank');
}
function exportMetadata(path) {
// Download metadata as an fs.meta.load-compatible .meta.gz
window.open((window.__BASE_PATH__ || '') + '/api/files/metadata?path=' + encodeURIComponent(path), '_blank');
}
function showFileProperties(path) {
// Fetch file properties and show in modal
fetch((window.__BASE_PATH__ || '') + '/api/files/properties?path=' + encodeURIComponent(path))
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -12,7 +12,7 @@ import (
func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
if grpcErr := pb.WithMasterClient(context.Background(), false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
if grpcErr := pb.WithMasterClient(false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
ClientType: clientType,
FilerGroup: filerGroup,
+1
View File
@@ -48,6 +48,7 @@ var Commands = []*Command{
cmdVolume,
cmdWebDav,
cmdSftp,
cmdNfs,
cmdWorker,
}
+1 -1
View File
@@ -596,7 +596,7 @@ var _ = filer_pb.FilerClient(&FileCopyWorker{})
func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
filerGrpcAddress := worker.filerAddress.ToGrpcAddress()
err = pb.WithGrpcClient(context.Background(), streamingMode, worker.signature, func(grpcConnection *grpc.ClientConn) error {
err = pb.WithGrpcClient(streamingMode, worker.signature, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, filerGrpcAddress, false, worker.options.grpcDialOption)
-28
View File
@@ -4,7 +4,6 @@ import (
"container/heap"
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -583,30 +582,3 @@ func BenchmarkConflictCheck(b *testing.B) {
})
}
}
// TestMetadataProcessorEmptyMarkerKeepsWatermarkStale: the MaxUnsyncedEvents
// marker (empty EventNotification, fresh timestamp) is dropped by AddSyncJob and
// does NOT advance processedTsWatermark, so offsetFunc keeps publishing the stale
// offset. This is why the client must not drive sync_offset off the watermark
// for these markers.
func TestMetadataProcessorEmptyMarkerKeepsWatermarkStale(t *testing.T) {
const staleOffset = int64(1_000_000_000)
freshTs := staleOffset + int64(time.Hour) // a "now"-ish source timestamp
p := NewMetadataProcessor(func(*filer_pb.SubscribeMetadataResponse) error { return nil }, 4, staleOffset)
marker := &filer_pb.SubscribeMetadataResponse{
TsNs: freshTs,
EventNotification: &filer_pb.EventNotification{},
}
if !filer_pb.IsEmpty(marker) {
t.Fatal("marker should be IsEmpty")
}
p.AddSyncJob(marker)
if got := p.processedTsWatermark.Load(); got != staleOffset {
t.Fatalf("empty marker advanced watermark to %d; want it to stay stale at %d", got, staleOffset)
}
t.Logf("marker carried fresh ts %d but watermark stayed stale at %d", freshTs, staleOffset)
}
+1 -1
View File
@@ -177,7 +177,7 @@ type simpleFilerClient struct {
}
func (c *simpleFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
return pb.WithGrpcClient(streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, c.grpcAddress.ToGrpcAddress(), false, c.grpcDialOption)
+16 -37
View File
@@ -6,7 +6,6 @@ import (
"io/fs"
"os"
"path"
"sort"
"strconv"
"strings"
@@ -37,15 +36,14 @@ var cmdFix = &Command{
}
var (
fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
fixIncludeDeleted = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
fixRemoteFile = cmdFix.Flag.Bool("remoteFile", false, "an optional, if true will not try to load the local .dat file, but only the remote file")
fixGenerateEcx = cmdFix.Flag.Bool("ecx", false, "regenerate a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards (missing shards are reconstructed from parity when enough survive). Run with the volume server stopped.")
fixEcDataShards = cmdFix.Flag.Int("ecDataShards", 0, "EC data shard count for -ecx (0 = read from .vif, otherwise default 10)")
fixEcParityShards = cmdFix.Flag.Int("ecParityShards", 0, "EC parity shard count for -ecx (0 = read from .vif, infer from shard count, otherwise default 4)")
fixEcUnsafeIgnoreSidecar = cmdFix.Flag.Bool("ecUnsafeIgnoreSidecar", false, "for -ecx: proceed even when the EC bitrot checksum sidecar (.ecsum) is malformed or stale, instead of failing closed; the reconstructed shards are not verified against it")
fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
fixIncludeDeleted = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
fixRemoteFile = cmdFix.Flag.Bool("remoteFile", false, "an optional, if true will not try to load the local .dat file, but only the remote file")
fixGenerateEcx = cmdFix.Flag.Bool("ecx", false, "regenerate a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards (missing shards are reconstructed from parity when enough survive). Run with the volume server stopped.")
fixEcDataShards = cmdFix.Flag.Int("ecDataShards", 0, "EC data shard count for -ecx (0 = read from .vif, otherwise default 10)")
fixEcParityShards = cmdFix.Flag.Int("ecParityShards", 0, "EC parity shard count for -ecx (0 = read from .vif, infer from shard count, otherwise default 4)")
)
type VolumeFileScanner4Fix struct {
@@ -192,34 +190,15 @@ func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) {
idxFile.Close()
}()
// Emit entries in .dat offset (append) order so the .idx stays the
// append-ordered log the volume server writes at runtime — not sorted by
// key, which is the .sdx / .ecx shape. A key-sorted .idx puts the
// highest-key needle last instead of the .dat-tail needle, which used to
// flip volumes read-only on load. Every tombstone is included (including
// any whose live needle is gone) so the last entry is the real .dat tail.
var values []needle_map.NeedleValue
collect := func(value needle_map.NeedleValue) error {
values = append(values, value)
return nil
}
if err = scaner.nm.AscendingVisit(collect); err != nil {
return scaner.nm.AscendingVisit(func(value needle_map.NeedleValue) error {
_, err := idxFile.Write(value.ToBytes())
if scaner.includeDeleted && err == nil {
if deleted, ok := scaner.nmDeleted.Get(value.Key); ok {
_, err = idxFile.Write(deleted.ToBytes())
}
}
return err
}
if scaner.includeDeleted {
if err = scaner.nmDeleted.AscendingVisit(collect); err != nil {
return err
}
}
sort.Slice(values, func(i, j int) bool {
return values[i].Offset.ToActualOffset() < values[j].Offset.ToActualOffset()
})
for _, value := range values {
if _, err = idxFile.Write(value.ToBytes()); err != nil {
return err
}
}
return nil
}
func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) {
@@ -382,7 +361,7 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
if !dataComplete {
ctx := &erasure_coding.ECContext{DataShards: dataShards, ParityShards: parityShards}
glog.Infof("volume %d: %d/%d shards present; reconstructing missing shards (%s) before index rebuild", volumeId, presentCount, dataShards+parityShards, ctx.String())
if _, err := erasure_coding.RebuildEcFiles(base, ctx, *fixEcUnsafeIgnoreSidecar); err != nil {
if _, err := erasure_coding.RebuildEcFilesWithContext(base, ctx); err != nil {
fail(fmt.Errorf("volume %d: reconstruct missing shards from %d survivors: %w", volumeId, presentCount, err))
return
}
+1 -1
View File
@@ -92,7 +92,7 @@ func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string) (base string
idxFile.Close()
nm.Close()
if _, err := erasure_coding.WriteEcFiles(base, erasure_coding.BackgroundECContext()); err != nil {
if err := erasure_coding.WriteEcFiles(base); err != nil {
t.Fatalf("WriteEcFiles: %v", err)
}
if err := erasure_coding.WriteSortedFileFromIdx(base, ".ecx"); err != nil {
-75
View File
@@ -1,75 +0,0 @@
package command
import (
"os"
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// TestSaveToIdxWritesOffsetOrder guards that weed fix rebuilds the .idx as an
// append-ordered log (sorted by .dat offset), not sorted by key. A key-sorted
// .idx puts the highest-key needle last instead of the .dat-tail needle, which
// flipped volumes read-only on load (issue #9688). It must also carry every
// tombstone — including one whose live needle is gone — so the last entry is
// the real .dat tail.
func TestSaveToIdxWritesOffsetOrder(t *testing.T) {
nm := needle_map.NewMemDb()
defer nm.Close()
nmDeleted := needle_map.NewMemDb()
defer nmDeleted.Close()
// Live needles with high keys at low offsets (as if written first).
for _, e := range []struct {
key uint64
offset int64
}{{30, 8}, {20, 128}, {10, 256}} {
if err := nm.Set(types.Uint64ToNeedleId(e.key), types.ToOffset(e.offset), types.Size(100)); err != nil {
t.Fatalf("nm.Set: %v", err)
}
}
// A tombstone at the .dat tail whose live needle is gone; the old SaveToIdx
// dropped these because the key was absent from the live map.
if err := nmDeleted.Set(types.Uint64ToNeedleId(5), types.ToOffset(384), types.TombstoneFileSize); err != nil {
t.Fatalf("nmDeleted.Set: %v", err)
}
scanner := &VolumeFileScanner4Fix{nm: nm, nmDeleted: nmDeleted, includeDeleted: true}
idxPath := filepath.Join(t.TempDir(), "v.idx")
if err := SaveToIdx(scanner, idxPath); err != nil {
t.Fatalf("SaveToIdx: %v", err)
}
f, err := os.Open(idxPath)
if err != nil {
t.Fatalf("open idx: %v", err)
}
defer f.Close()
var offsets []int64
var lastKey types.NeedleId
if err := idx.WalkIndexFile(f, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
offsets = append(offsets, offset.ToActualOffset())
lastKey = key
return nil
}); err != nil {
t.Fatalf("walk idx: %v", err)
}
if len(offsets) != 4 {
t.Fatalf("expected 4 entries (3 live + 1 tombstone), got %d", len(offsets))
}
for i := 1; i < len(offsets); i++ {
if offsets[i] < offsets[i-1] {
t.Fatalf("entries not in offset order: %v", offsets)
}
}
// The .dat-tail needle (the orphan tombstone at the highest offset) must be last.
if lastKey != types.Uint64ToNeedleId(5) {
t.Errorf("last entry should be the .dat-tail tombstone (key 5), got key %v", lastKey)
}
}
+2 -41
View File
@@ -4,7 +4,6 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"math/bits"
@@ -1788,19 +1787,8 @@ func printWelcomeMessage() {
fmt.Fprintf(&sb, " Admin UI: http://%s:%d\n", *miniIp, *miniAdminOptions.port)
}
fmt.Fprintf(&sb, "\n Data Directory: %s\n", *miniDataFolders)
firstDir := util.StringSplit(*miniDataFolders, ",")[0]
if ds := stats_collect.NewDiskStatus(firstDir); ds != nil && ds.All > 0 {
fmt.Fprintf(&sb, " Free Space: %s\n", util.BytesToHumanReadable(ds.Free))
}
if miniMasterOptions.volumeSizeLimitMB != nil {
fmt.Fprintf(&sb, " Volume Size: %s\n", util.BytesToHumanReadable(uint64(*miniMasterOptions.volumeSizeLimitMB)*bytesPerMB))
}
if max, free, ok := miniVolumeCounts(); ok {
fmt.Fprintf(&sb, " Volume Count: %d\n", max)
fmt.Fprintf(&sb, " Free Volumes: %d\n", free)
}
sb.WriteString("\n Press Ctrl+C to stop all components")
fmt.Fprintf(&sb, "\n Data Directory: %s\n\n", *miniDataFolders)
sb.WriteString(" Press Ctrl+C to stop all components")
switch {
case s3api.HasAnyIdentity():
@@ -1822,33 +1810,6 @@ func printWelcomeMessage() {
fmt.Println("")
}
// miniVolumeCounts asks the local master for the total volume slots the data
// directory supports (Topology.Max) and how many are still free (Topology.Free).
// Best-effort: any error returns ok=false so the welcome banner simply omits
// the lines.
func miniVolumeCounts() (max, free int64, ok bool) {
url := getHealthCheckAddr(fmt.Sprintf("http://%s:%d/dir/status", *miniIp, *miniMasterOptions.port))
client := &http.Client{Timeout: 2 * time.Second}
resp, err := client.Get(url)
if err != nil {
return 0, 0, false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0, 0, false
}
var status struct {
Topology struct {
Max int64
Free int64
}
}
if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
return 0, 0, false
}
return status.Topology.Max, status.Topology.Free, true
}
// ensureMiniBuckets creates each named bucket on the embedded filer if it does
// not already exist. bucketSpec is a comma-separated list (whitespace around
// each name is trimmed); empty entries and an empty spec are no-ops so callers
+127
View File
@@ -0,0 +1,127 @@
package command
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server_nfs "github.com/seaweedfs/seaweedfs/weed/server/nfs"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/version"
)
var (
nfsStandaloneOptions NfsOptions
)
type NfsOptions struct {
filer *string
ipBind *string
port *int
filerRootPath *string
readOnly *bool
allowedClients *string
volumeServerAccess *string
portmapBind *string
}
func init() {
cmdNfs.Run = runNfs // break init cycle
nfsStandaloneOptions.filer = cmdNfs.Flag.String("filer", "localhost:8888", "filer server address")
nfsStandaloneOptions.ipBind = cmdNfs.Flag.String("ip.bind", "127.0.0.1", "ip address to bind to. Defaults to loopback; override explicitly to expose the experimental server to the network.")
nfsStandaloneOptions.port = cmdNfs.Flag.Int("port", 2049, "NFS server listen port")
nfsStandaloneOptions.filerRootPath = cmdNfs.Flag.String("filer.path", "", "remote path from filer server to export. Required: no default is provided so operators must opt in to exporting a namespace subtree.")
nfsStandaloneOptions.readOnly = cmdNfs.Flag.Bool("readOnly", false, "export the filer path as read only")
nfsStandaloneOptions.allowedClients = cmdNfs.Flag.String("allowedClients", "", "comma-separated client IPs, hostnames, or CIDRs allowed to connect")
nfsStandaloneOptions.volumeServerAccess = cmdNfs.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
nfsStandaloneOptions.portmapBind = cmdNfs.Flag.String("portmap.bind", "", "when set, bind a built-in portmap v2 responder on <ip>:111 so plain `mount -t nfs` works without client-side portmap bypass. Empty disables it. Binding port 111 requires root or CAP_NET_BIND_SERVICE and must not conflict with a system rpcbind.")
}
var cmdNfs = &Command{
UsageLine: "nfs -port=2049 -filer=<ip:port> -filer.path=<exported subtree>",
Short: "start an experimental NFSv3 server backed by a filer",
Long: `start an experimental NFSv3 server backed by a filer.
This command serves an experimental filer-native NFSv3 frontend with
deterministic filehandles, filer-backed metadata operations, and direct
volume-server data access for chunk reads and buffered writes.
Safer defaults (since export ACLs are still not implemented):
- ip.bind defaults to 127.0.0.1, so the server is not reachable from
other hosts unless you override it explicitly.
- filer.path has no default; you must pick the subtree to export.
Override -ip.bind to a routable address only after you have reviewed
-allowedClients and the readiness of the rest of your deployment.
Mounting from a Linux client
----------------------------
The server does not run portmap/rpcbind by default. That means Linux
mount.nfs, which queries portmap on port 111 first, will fail with
"portmap query failed" against the plain form:
mount -t nfs -o nfsvers=3,nolock <host>:/export /mnt
Either tell the client to bypass portmap:
mount -t nfs -o nfsvers=3,nolock,port=2049,mountport=2049,\
proto=tcp,mountproto=tcp <host>:/export /mnt
or enable the built-in portmap responder on the server:
weed nfs ... -portmap.bind=0.0.0.0
With the responder enabled MOUNT v3 is answered over both TCP and UDP,
so the plain mount form above just works — no mountproto override is
required even on clients whose default mountproto is UDP.
Binding port 111 requires root or CAP_NET_BIND_SERVICE and must not
collide with a system rpcbind.
`,
}
func runNfs(cmd *Command, args []string) bool {
util.LoadSecurityConfiguration()
if *nfsStandaloneOptions.ipBind == "" {
*nfsStandaloneOptions.ipBind = "127.0.0.1"
}
if *nfsStandaloneOptions.filerRootPath == "" {
glog.Errorf("-filer.path is required: pick an explicit subtree to export; exporting \"/\" is not a default")
return false
}
if *nfsStandaloneOptions.filerRootPath == "/" {
glog.Warningf("-filer.path=/ exports the entire filer namespace; ensure -allowedClients or -ip.bind constrains access")
}
listenAddress := fmt.Sprintf("%s:%d", *nfsStandaloneOptions.ipBind, *nfsStandaloneOptions.port)
glog.V(0).Infof("Starting Seaweed NFS Server %s at %s", version.Version(), listenAddress)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
nfsServer, err := weed_server_nfs.NewServer(&weed_server_nfs.Option{
Filer: pb.ServerAddress(*nfsStandaloneOptions.filer),
BindIp: *nfsStandaloneOptions.ipBind,
Port: *nfsStandaloneOptions.port,
FilerRootPath: *nfsStandaloneOptions.filerRootPath,
ReadOnly: *nfsStandaloneOptions.readOnly,
AllowedClients: util.StringSplit(*nfsStandaloneOptions.allowedClients, ","),
VolumeServerAccess: *nfsStandaloneOptions.volumeServerAccess,
GrpcDialOption: grpcDialOption,
PortmapBind: *nfsStandaloneOptions.portmapBind,
})
if err != nil {
glog.Errorf("NFS Server startup error: %v", err)
return false
}
if err := nfsServer.Start(); err != nil {
glog.Errorf("NFS Server startup error: %v", err)
return false
}
return true
}
+1 -1
View File
@@ -145,7 +145,7 @@ func runUpload(cmd *Command, args []string) bool {
}
func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress pb.ServerAddress) (replication string, err error) {
err = pb.WithMasterClient(context.Background(), false, masterAddress, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
err = pb.WithMasterClient(false, masterAddress, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return fmt.Errorf("get master %s configuration: %v", masterAddress, err)
+2 -2
View File
@@ -21,7 +21,7 @@ func runVersion(cmd *Command, args []string) bool {
fmt.Printf("version %s %s %s\n", version.Version(), runtime.GOOS, runtime.GOARCH)
println()
println("For enterprise users, please visit https://seaweedfs.com for the SeaweedFS Enterprise Edition,")
println("which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.")
println("For enterprise users, please visit https://seaweedfs.com for SeaweedFS Enterprise Edition,")
println("which has a self-healing storage format with better data protection.")
return true
}
-22
View File
@@ -23,7 +23,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/server/constants"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
@@ -140,9 +139,6 @@ var (
volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
minFreeSpace = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
ecBitrotChecksum = cmdVolume.Flag.Bool("ec.bitrotChecksum", true, "write a bitrot checksum sidecar (.ecsum) when generating EC shards, enabling silent-corruption detection on scrub/rebuild")
ecBitrotBlockSizeMB = cmdVolume.Flag.Int("ec.bitrotBlockSizeMB", 16, "EC bitrot checksum block granularity in MiB; must be a power of two and at least 1")
)
func runVolume(cmd *Command, args []string) bool {
@@ -175,24 +171,6 @@ func runVolume(cmd *Command, args []string) bool {
*v.mastersString = *v.mserverString
}
// Apply EC bitrot checksum settings.
erasure_coding.BitrotProtectionEnabled = *ecBitrotChecksum
// Bound-check before the multiply so a huge value cannot overflow int64 past
// the power-of-two check. Cap = shared MaxBitrotBlockSize, kept in sync with
// ValidateBitrotManifest.
const maxBitrotBlockSizeMB = erasure_coding.MaxBitrotBlockSize / (1024 * 1024)
if mb := *ecBitrotBlockSizeMB; mb >= 1 && mb <= maxBitrotBlockSizeMB {
if blockSize := int64(mb) * 1024 * 1024; blockSize&(blockSize-1) == 0 {
erasure_coding.BitrotBlockSize = blockSize
} else if mb != 16 {
glog.Warningf("ignoring invalid -ec.bitrotBlockSizeMB=%d (must be a power of two); using %d MiB",
mb, erasure_coding.BitrotBlockSize/(1024*1024))
}
} else if *ecBitrotBlockSizeMB != 16 {
glog.Warningf("ignoring out-of-range -ec.bitrotBlockSizeMB=%d (must be a power of two in [1, %d] MiB); using %d MiB",
*ecBitrotBlockSizeMB, maxBitrotBlockSizeMB, erasure_coding.BitrotBlockSize/(1024*1024))
}
minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
+1 -1
View File
@@ -98,7 +98,7 @@ func (store *IamGrpcStore) withIamClient(ctx context.Context, fn func(ctx contex
}
}
return pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error {
return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
return fn(ctx, client)
}, filerAddress.ToGrpcAddress(), false, dialOption)
+1 -1
View File
@@ -86,7 +86,7 @@ func (s *PropagatingCredentialStore) propagateChange(ctx context.Context, fn fun
wg.Add(1)
go func(server string) {
defer wg.Done()
err := pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error {
err := pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
glog.V(4).Infof("IAM: successfully connected to S3 server %s for propagation", server)
client := s3_pb.NewSeaweedS3IamCacheClient(conn)
return fn(propagateCtx, client)
@@ -21,7 +21,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by
dirStr, dirHash, name := GenDirAndName(key)
_, err = db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
res, err := db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
if err == nil {
return
}
@@ -34,7 +34,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by
// now the insert failed possibly due to duplication constraints
glog.V(1).InfofCtx(ctx, "kv insert falls back to update: %s", err)
res, err := db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr)
res, err = db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr)
if err != nil {
return fmt.Errorf("kv upsert: %s", err)
}
+3
View File
@@ -13,6 +13,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/filer/empty_folder_cleanup"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -63,6 +64,7 @@ type Filer struct {
DeletionRetryQueue *DeletionRetryQueue
EmptyFolderCleaner *empty_folder_cleanup.EmptyFolderCleaner
EmptyFolderCleanupDelay time.Duration
inodeSequencer sequence.Sequencer
}
func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
@@ -77,6 +79,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
MaxFilenameLength: maxFilenameLength,
deletionQuit: make(chan struct{}),
DeletionRetryQueue: NewDeletionRetryQueue(),
inodeSequencer: newInodeSequencer(filerHost),
}
if f.UniqueFilerId < 0 {
f.UniqueFilerId = -f.UniqueFilerId
-29
View File
@@ -233,35 +233,6 @@ func ClonePathConf(src *filer_pb.FilerConf_PathConf) *filer_pb.FilerConf_PathCon
}
}
// ApplyBucketQuotaReadOnly sets read-only when usedSize exceeds quota and clears it
// once back under, reporting whether the flag changed. A non-positive quota is left
// untouched so a manually locked bucket is never reopened.
func (fc *FilerConf) ApplyBucketQuotaReadOnly(locationPrefix string, usedSize, quota float64) (readOnly, changed bool) {
if quota <= 0 {
return fc.MatchStorageRule(locationPrefix).ReadOnly, false
}
locConf := ClonePathConf(fc.MatchStorageRule(locationPrefix))
locConf.LocationPrefix = locationPrefix
wasReadOnly := locConf.ReadOnly
if wasReadOnly {
if usedSize < quota {
locConf.ReadOnly = false
}
} else {
if usedSize > quota {
locConf.ReadOnly = true
}
}
if locConf.ReadOnly == wasReadOnly {
return wasReadOnly, false
}
fc.SetLocationConf(locConf)
return locConf.ReadOnly, true
}
func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) {
ttls = make(map[string]string)
fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
-33
View File
@@ -128,36 +128,3 @@ func TestClonePathConfNil(t *testing.T) {
assert.NotNil(t, clone, "ClonePathConf(nil) should return a non-nil empty PathConf")
assert.Equal(t, "", clone.LocationPrefix, "ClonePathConf(nil) should return empty PathConf")
}
func TestApplyBucketQuotaReadOnly(t *testing.T) {
const prefix = "/buckets/b/"
// over quota: flips to read-only
fc := NewFilerConf()
readOnly, changed := fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
assert.True(t, changed)
assert.True(t, readOnly)
assert.True(t, fc.MatchStorageRule(prefix).ReadOnly)
// still over quota: no change
_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
assert.False(t, changed)
// back under quota: flips to writable
readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
assert.True(t, changed)
assert.False(t, readOnly)
assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
// quota disabled leaves the flag untouched, so manual locks survive
fc = NewFilerConf()
fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, -1)
assert.False(t, changed)
assert.True(t, readOnly)
// under quota and not read-only: no rule churn
fc = NewFilerConf()
_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
assert.False(t, changed)
}
+40 -15
View File
@@ -1,26 +1,51 @@
package filer
import (
"time"
"os"
"strconv"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
)
// ensureEntryInode derives a stable inode the same way the FUSE mount does, so
// the filer-stored value matches what a mount would otherwise compute and no
// per-object reverse index is required. Hard links hash their shared
// HardLinkId, so every link resolves to one inode; other entries hash the path
// and creation time.
// newInodeSequencer constructs the inode sequencer used to assign object
// identity for filer entries. The Snowflake node id defaults to a masked hash
// of filerHost, which only has 1024 possible values; operators running a
// multi-filer cluster should set SEAWEEDFS_FILER_SNOWFLAKE_ID to an explicit
// per-filer value (1..1023) to avoid birthday-paradox collisions.
//
// Initialization failures are fatal: a process-local fallback allocator would
// re-use inode values across restarts and violate the stable object identity
// guarantee that NFS filehandles and the inode secondary index rely on.
func newInodeSequencer(filerHost pb.ServerAddress) sequence.Sequencer {
snowflakeId := parseSnowflakeIdFromEnv()
seq, err := sequence.NewSnowflakeSequencer(string(filerHost), snowflakeId)
if err != nil {
glog.Fatalf("initialize inode sequencer for filer %s (snowflakeId=%d): %v", filerHost, snowflakeId, err)
}
return seq
}
func parseSnowflakeIdFromEnv() int {
raw := os.Getenv("SEAWEEDFS_FILER_SNOWFLAKE_ID")
if raw == "" {
return 0
}
id, err := strconv.Atoi(raw)
if err != nil || id < 0 || id > 0x3ff {
glog.Fatalf("SEAWEEDFS_FILER_SNOWFLAKE_ID must be an integer in [0,1023], got %q", raw)
}
return id
}
func (f *Filer) ensureEntryInode(entry *Entry) {
if entry == nil || entry.Attr.Inode != 0 {
return
}
if entry.Attr.Crtime.IsZero() {
entry.Attr.Crtime = time.Now()
}
if len(entry.HardLinkId) > 0 {
entry.Attr.Inode = uint64(util.HashStringToLong(string(entry.HardLinkId)))
return
}
entry.Attr.Inode = entry.FullPath.AsInode(entry.Attr.Crtime.Unix())
entry.Attr.Inode = f.nextInode()
}
func (f *Filer) nextInode() uint64 {
return f.inodeSequencer.NextFileId(1)
}
+300
View File
@@ -0,0 +1,300 @@
package filer
import (
"context"
"encoding/binary"
"encoding/json"
"sort"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const inodeIndexKeyPrefix = "filer.inode.path."
const InodeIndexInitialGeneration uint64 = 1
type inodeIndexEntry struct {
path util.FullPath
inode uint64
}
type InodeIndexRecord struct {
Generation uint64 `json:"generation,omitempty"`
Paths []string `json:"paths,omitempty"`
}
func InodeIndexKey(inode uint64) []byte {
key := make([]byte, len(inodeIndexKeyPrefix)+8)
copy(key, inodeIndexKeyPrefix)
binary.BigEndian.PutUint64(key[len(inodeIndexKeyPrefix):], inode)
return key
}
func DecodeInodeIndexRecord(value []byte) (*InodeIndexRecord, error) {
if len(value) == 0 {
return &InodeIndexRecord{}, nil
}
// The first foundation slice stored the current path as raw bytes. Keep that
// format readable so existing records are transparently upgraded on write.
if value[0] != '{' {
record := &InodeIndexRecord{Generation: InodeIndexInitialGeneration}
record.addPath(util.FullPath(value))
return record, nil
}
record := &InodeIndexRecord{}
if err := json.Unmarshal(value, record); err != nil {
return nil, err
}
record.normalize()
return record, nil
}
func (record *InodeIndexRecord) Encode() ([]byte, error) {
record.normalize()
return json.Marshal(record)
}
func (record *InodeIndexRecord) normalize() {
if len(record.Paths) == 0 {
return
}
if record.Generation == 0 {
record.Generation = InodeIndexInitialGeneration
}
sanitized := make([]string, 0, len(record.Paths))
for _, path := range record.Paths {
if path == "" {
continue
}
sanitized = append(sanitized, path)
}
if len(sanitized) == 0 {
record.Paths = nil
return
}
sort.Strings(sanitized)
deduped := sanitized[:1]
for _, path := range sanitized[1:] {
if path == deduped[len(deduped)-1] {
continue
}
deduped = append(deduped, path)
}
record.Paths = deduped
}
func (record *InodeIndexRecord) addPath(path util.FullPath) bool {
if path == "" {
return false
}
record.normalize()
target := string(path)
index := sort.SearchStrings(record.Paths, target)
if index < len(record.Paths) && record.Paths[index] == target {
return false
}
record.Paths = append(record.Paths, "")
copy(record.Paths[index+1:], record.Paths[index:])
record.Paths[index] = target
return true
}
func (record *InodeIndexRecord) removePath(path util.FullPath) bool {
if len(record.Paths) == 0 || path == "" {
return false
}
record.normalize()
target := string(path)
index := sort.SearchStrings(record.Paths, target)
if index >= len(record.Paths) || record.Paths[index] != target {
return false
}
record.Paths = append(record.Paths[:index], record.Paths[index+1:]...)
if len(record.Paths) == 0 {
record.Paths = nil
}
return true
}
func (record *InodeIndexRecord) CanonicalPath() util.FullPath {
record.normalize()
if len(record.Paths) == 0 {
return ""
}
return util.FullPath(record.Paths[0])
}
func (record *InodeIndexRecord) FullPaths() []util.FullPath {
record.normalize()
if len(record.Paths) == 0 {
return nil
}
paths := make([]util.FullPath, 0, len(record.Paths))
for _, path := range record.Paths {
paths = append(paths, util.FullPath(path))
}
return paths
}
func (fsw *FilerStoreWrapper) lookupInodeIndex(ctx context.Context, inode uint64) (*InodeIndexRecord, error) {
if inode == 0 {
return nil, ErrKvNotFound
}
value, err := fsw.KvGet(ctx, InodeIndexKey(inode))
if err != nil {
return nil, err
}
return DecodeInodeIndexRecord(value)
}
func (fsw *FilerStoreWrapper) storeInodeIndex(ctx context.Context, path util.FullPath, inode uint64) error {
if inode == 0 || path == "" {
return nil
}
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
if err != ErrKvNotFound {
return err
}
record = &InodeIndexRecord{Generation: InodeIndexInitialGeneration}
}
record.addPath(path)
value, err := record.Encode()
if err != nil {
return err
}
return fsw.KvPut(ctx, InodeIndexKey(inode), value)
}
func (fsw *FilerStoreWrapper) lookupInodePath(ctx context.Context, inode uint64) (util.FullPath, error) {
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
return "", err
}
path := record.CanonicalPath()
if path == "" {
return "", ErrKvNotFound
}
return path, nil
}
func (fsw *FilerStoreWrapper) lookupInodePaths(ctx context.Context, inode uint64) ([]util.FullPath, error) {
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
return nil, err
}
paths := record.FullPaths()
if len(paths) == 0 {
return nil, ErrKvNotFound
}
return paths, nil
}
func (fsw *FilerStoreWrapper) removePathFromInodeIndex(ctx context.Context, path util.FullPath, inode uint64) error {
if inode == 0 || path == "" {
return nil
}
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
if err == ErrKvNotFound {
return nil
}
return err
}
if !record.removePath(path) {
return nil
}
if len(record.Paths) == 0 {
return fsw.KvDelete(ctx, InodeIndexKey(inode))
}
value, err := record.Encode()
if err != nil {
return err
}
return fsw.KvPut(ctx, InodeIndexKey(inode), value)
}
func (fsw *FilerStoreWrapper) collectInodeIndexEntries(ctx context.Context, dirPath util.FullPath) ([]inodeIndexEntry, error) {
// Honor caller cancellation during the walk: a DeleteFolderChildren on a
// pathological directory could otherwise loop indefinitely gathering
// entries even after the client has given up, turning into a DoS vector.
// If the walk is aborted, the caller treats the index cleanup as
// best-effort and drops the partial result.
var collected []inodeIndexEntry
if err := fsw.collectInodeIndexEntriesRecursive(ctx, dirPath, &collected); err != nil {
return nil, err
}
return collected, nil
}
func (fsw *FilerStoreWrapper) collectInodeIndexEntriesRecursive(ctx context.Context, dirPath util.FullPath, collected *[]inodeIndexEntry) error {
actualStore := fsw.getActualStore(dirPath + "/")
lastFileName := ""
includeStartFile := false
for {
page := make([]*Entry, 0, PaginationSize)
nextLastFileName, err := actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, PaginationSize, func(entry *Entry) (bool, error) {
page = append(page, entry)
return true, nil
})
if err != nil {
return err
}
for _, entry := range page {
if entry.Attr.Inode != 0 {
*collected = append(*collected, inodeIndexEntry{path: entry.FullPath, inode: entry.Attr.Inode})
}
if entry.IsDirectory() {
if err := fsw.collectInodeIndexEntriesRecursive(ctx, entry.FullPath, collected); err != nil {
return err
}
}
}
if len(page) < PaginationSize {
return nil
}
lastFileName = nextLastFileName
includeStartFile = false
}
}
// recordInodeIndexWrite updates the inode→path secondary index after the
// primary store mutation has already succeeded. The index is best-effort: a
// failure here must not surface as an operation error, because the caller
// would then observe a failed create/update even though the entry was
// persisted, and a retry cannot heal the index (DeleteEntry exits early once
// the entry is gone). We log and let later writes rebuild the record.
func (fsw *FilerStoreWrapper) recordInodeIndexWrite(ctx context.Context, op string, path util.FullPath, inode uint64) {
if inode == 0 || path == "" {
return
}
if err := fsw.storeInodeIndex(ctx, path, inode); err != nil {
glog.WarningfCtx(ctx, "%s: update inode index for %s (inode %d): %v", op, path, inode, err)
}
}
// recordInodeIndexRemoval mirrors recordInodeIndexWrite for removals.
func (fsw *FilerStoreWrapper) recordInodeIndexRemoval(ctx context.Context, op string, path util.FullPath, inode uint64) {
if inode == 0 || path == "" {
return
}
if err := fsw.removePathFromInodeIndex(ctx, path, inode); err != nil {
glog.WarningfCtx(ctx, "%s: clear inode index for %s (inode %d): %v", op, path, inode, err)
}
}
+206
View File
@@ -0,0 +1,206 @@
package filer
import (
"context"
"os"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFilerStoreWrapperMaintainsInodeIndexLifecycle(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
created := &Entry{
FullPath: util.FullPath("/docs/report.txt"),
Attr: Attr{
Mode: 0o644,
Inode: 42,
},
}
require.NoError(t, wrapper.InsertEntry(ctx, created))
path, err := wrapper.lookupInodePath(ctx, created.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, created.FullPath, path)
paths, err := wrapper.lookupInodePaths(ctx, created.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{created.FullPath}, paths)
record, err := wrapper.lookupInodeIndex(ctx, created.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, InodeIndexInitialGeneration, record.Generation)
updated := &Entry{
FullPath: util.FullPath("/docs/report.txt"),
Attr: Attr{
Mode: 0o600,
Inode: 42,
},
}
require.NoError(t, wrapper.UpdateEntry(ctx, updated))
path, err = wrapper.lookupInodePath(ctx, updated.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, updated.FullPath, path)
require.NoError(t, wrapper.DeleteEntry(ctx, created.FullPath))
_, err = wrapper.lookupInodePath(ctx, created.Attr.Inode)
require.ErrorIs(t, err, ErrKvNotFound)
}
func TestFilerStoreWrapperMaintainsMultiplePathsPerInode(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
inode := uint64(88)
hardLinkId := NewHardLinkId()
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/links/b.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
HardLinkId: hardLinkId,
HardLinkCounter: 2,
}))
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/links/a.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
HardLinkId: hardLinkId,
HardLinkCounter: 2,
}))
paths, err := wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/links/a.txt", "/links/b.txt"}, paths)
record, err := wrapper.lookupInodeIndex(ctx, inode)
require.NoError(t, err)
assert.Equal(t, InodeIndexInitialGeneration, record.Generation)
path, err := wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/links/a.txt"), path)
require.NoError(t, wrapper.DeleteEntry(ctx, util.FullPath("/links/a.txt")))
paths, err = wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/links/b.txt"}, paths)
path, err = wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/links/b.txt"), path)
}
func TestFilerStoreWrapperUpgradesLegacySinglePathInodeIndexRecords(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
inode := uint64(91)
require.NoError(t, wrapper.KvPut(ctx, InodeIndexKey(inode), []byte("/legacy/path.txt")))
path, err := wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/legacy/path.txt"), path)
paths, err := wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/legacy/path.txt"}, paths)
require.NoError(t, wrapper.storeInodeIndex(ctx, util.FullPath("/legacy/second.txt"), inode))
paths, err = wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/legacy/path.txt", "/legacy/second.txt"}, paths)
value, err := wrapper.KvGet(ctx, InodeIndexKey(inode))
require.NoError(t, err)
assert.JSONEq(t, `{"generation":1,"paths":["/legacy/path.txt","/legacy/second.txt"]}`, string(value))
}
func TestFilerStoreWrapperKeepsInodeIndexWhenDeleteArrivesAfterRenameInsert(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
inode := uint64(77)
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/old/name.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
}))
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/new/name.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
}))
require.NoError(t, wrapper.DeleteEntry(ctx, util.FullPath("/old/name.txt")))
path, err := wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/new/name.txt"), path)
paths, err := wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/new/name.txt"}, paths)
}
func TestRecursiveDeleteRemovesDescendantInodeIndexes(t *testing.T) {
f, store := newTestFilerWithStubStore()
ctx := context.Background()
entries := []*Entry{
{
FullPath: util.FullPath("/tree"),
Attr: Attr{
Mode: os.ModeDir | 0o755,
Inode: 100,
},
},
{
FullPath: util.FullPath("/tree/file.txt"),
Attr: Attr{
Mode: 0o644,
Inode: 101,
},
},
{
FullPath: util.FullPath("/tree/subdir"),
Attr: Attr{
Mode: os.ModeDir | 0o755,
Inode: 102,
},
},
{
FullPath: util.FullPath("/tree/subdir/nested.txt"),
Attr: Attr{
Mode: 0o644,
Inode: 103,
},
},
}
for _, entry := range entries {
require.NoError(t, f.Store.InsertEntry(ctx, entry))
}
require.NoError(t, f.DeleteEntryMetaAndData(ctx, util.FullPath("/tree"), true, false, false, false, nil, 0))
for _, inode := range []uint64{100, 101, 102, 103} {
_, err := f.Store.(*FilerStoreWrapper).lookupInodePath(ctx, inode)
require.ErrorIs(t, err, ErrKvNotFound)
}
for _, path := range []string{"/tree", "/tree/file.txt", "/tree/subdir", "/tree/subdir/nested.txt"} {
_, err := store.FindEntry(ctx, util.FullPath(path))
require.Error(t, err)
}
}
-42
View File
@@ -4,7 +4,6 @@ import (
"context"
"os"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -12,47 +11,6 @@ import (
"github.com/stretchr/testify/require"
)
func TestEnsureEntryInodeMatchesFuseDerivation(t *testing.T) {
f := &Filer{}
crtime := time.Unix(1700000000, 0)
entry := &Entry{
FullPath: util.FullPath("/dir/file.txt"),
Attr: Attr{Crtime: crtime},
}
f.ensureEntryInode(entry)
// The filer stores exactly what the FUSE mount would compute for a
// non-hard-linked entry, and it is deterministic across calls.
assert.Equal(t, entry.FullPath.AsInode(crtime.Unix()), entry.Attr.Inode)
again := &Entry{FullPath: entry.FullPath, Attr: Attr{Crtime: crtime}}
f.ensureEntryInode(again)
assert.Equal(t, entry.Attr.Inode, again.Attr.Inode)
}
func TestEnsureEntryInodeSharesAcrossHardLinks(t *testing.T) {
f := &Filer{}
hardLinkId := NewHardLinkId()
a := &Entry{
FullPath: util.FullPath("/links/a.txt"),
Attr: Attr{Crtime: time.Unix(1700000000, 0)},
HardLinkId: hardLinkId,
}
b := &Entry{
FullPath: util.FullPath("/links/b.txt"),
Attr: Attr{Crtime: time.Unix(1800000000, 0)},
HardLinkId: hardLinkId,
}
f.ensureEntryInode(a)
f.ensureEntryInode(b)
// Every link to the same target resolves to one inode, independent of path
// or creation time.
assert.Equal(t, uint64(util.HashStringToLong(string(hardLinkId))), a.Attr.Inode)
assert.Equal(t, a.Attr.Inode, b.Attr.Inode)
}
func newTestFilerWithStubStore() (*Filer, *stubFilerStore) {
store := newStubFilerStore()
f := NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+1
View File
@@ -276,6 +276,7 @@ func newTestFiler(t *testing.T, store *stubFilerStore, rs *FilerRemoteStorage) *
FilerConf: NewFilerConf(),
MaxFilenameLength: 255,
MasterClient: mc,
inodeSequencer: newInodeSequencer("test-filer"),
fileIdDeletionQueue: util.NewUnboundedQueue(),
deletionQuit: make(chan struct{}),
LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
+46 -6
View File
@@ -132,6 +132,7 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := entry.FullPath
actualStore := fsw.getActualStore(entry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
start := time.Now()
@@ -151,7 +152,11 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
return err
}
return actualStore.InsertEntry(ctx, entry)
if err := actualStore.InsertEntry(ctx, entry); err != nil {
return err
}
fsw.recordInodeIndexWrite(ctx, "InsertEntry", fullPath, entry.Attr.Inode)
return nil
}
// InsertEntryKnownAbsent skips the pre-insert FindEntry path when the caller has
@@ -161,6 +166,7 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := entry.FullPath
actualStore := fsw.getActualStore(entry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
start := time.Now()
@@ -179,7 +185,11 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
}
}
return actualStore.InsertEntry(ctx, entry)
if err := actualStore.InsertEntry(ctx, entry); err != nil {
return err
}
fsw.recordInodeIndexWrite(ctx, "InsertEntryKnownAbsent", fullPath, entry.Attr.Inode)
return nil
}
func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
@@ -187,6 +197,7 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := entry.FullPath
actualStore := fsw.getActualStore(entry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "update").Inc()
start := time.Now()
@@ -206,7 +217,11 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
return err
}
return actualStore.UpdateEntry(ctx, entry)
if err := actualStore.UpdateEntry(ctx, entry); err != nil {
return err
}
fsw.recordInodeIndexWrite(ctx, "UpdateEntry", fullPath, entry.Attr.Inode)
return nil
}
func normalizeEntryMimeForStore(entry *Entry) {
@@ -258,6 +273,8 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
if findErr == filer_pb.ErrNotFound || existingEntry == nil {
return nil
}
inode := existingEntry.Attr.Inode
fullPath := existingEntry.FullPath
if len(existingEntry.HardLinkId) != 0 {
// remove hard link
op := ctx.Value("OP")
@@ -272,7 +289,11 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
}
}
return actualStore.DeleteEntry(ctx, fp)
if err := actualStore.DeleteEntry(ctx, fp); err != nil {
return err
}
fsw.recordInodeIndexRemoval(ctx, "DeleteEntry", fullPath, inode)
return nil
}
func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
@@ -280,6 +301,8 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := existingEntry.FullPath
inode := existingEntry.Attr.Inode
actualStore := fsw.getActualStore(existingEntry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
start := time.Now()
@@ -302,7 +325,11 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
}
}
return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
if err := actualStore.DeleteEntry(ctx, existingEntry.FullPath); err != nil {
return err
}
fsw.recordInodeIndexRemoval(ctx, "DeleteOneEntry", fullPath, inode)
return nil
}
func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
@@ -317,7 +344,20 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.
stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Observe(time.Since(start).Seconds())
}()
return actualStore.DeleteFolderChildren(ctx, fp)
collected, err := fsw.collectInodeIndexEntries(ctx, fp)
if err != nil {
// Index collection is best-effort: a failure here only prevents inode
// index housekeeping, not the directory removal itself.
glog.WarningfCtx(ctx, "collectInodeIndexEntries %s: %v; deleting folder children without index cleanup", fp, err)
collected = nil
}
if err := actualStore.DeleteFolderChildren(ctx, fp); err != nil {
return err
}
for _, entry := range collected {
fsw.recordInodeIndexRemoval(ctx, "DeleteFolderChildren", entry.path, entry.inode)
}
return nil
}
func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {
-7
View File
@@ -13,13 +13,6 @@ type SqlGenMysql struct {
UpsertQueryTemplate string
}
// DefaultUpsertQuery keeps INSERTs idempotent so the inode-index KvPut
// after every entry write does not emit a duplicate-key roundtrip on
// every mutation. The VALUES() form works on MariaDB and MySQL >=5.7;
// the newer "AS new" alias from MySQL 8.0.19 errors on MariaDB, so it
// is left for users to opt into via an explicit upsertQuery.
const DefaultUpsertQuery = "INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE `meta` = VALUES(`meta`)"
var (
_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
)
-28
View File
@@ -1,28 +0,0 @@
package mysql
import (
"strings"
"testing"
)
func TestDefaultUpsertQueryUsesOnDuplicateKey(t *testing.T) {
gen := &SqlGenMysql{UpsertQueryTemplate: DefaultUpsertQuery}
got := gen.GetSqlInsert("filemeta")
if !strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("expected ON DUPLICATE KEY UPDATE in default upsert, got: %s", got)
}
if strings.Contains(got, "AS `new`") {
t.Fatalf("default should avoid MySQL 8.0.19 row-alias syntax for MariaDB compat, got: %s", got)
}
if !strings.Contains(got, "`filemeta`") {
t.Fatalf("expected backticked table name, got: %s", got)
}
}
func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
gen := &SqlGenMysql{}
got := gen.GetSqlInsert("filemeta")
if strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("plain INSERT path should not contain ON DUPLICATE KEY UPDATE, got: %s", got)
}
}
-5
View File
@@ -35,9 +35,6 @@ func (store *MysqlStore) GetName() string {
func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs avoid the duplicate-key roundtrip the
// inode-index KvPut would otherwise emit on every write.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"dsn"),
configuration.GetString(prefix+"upsertQuery"),
@@ -67,8 +64,6 @@ func (store *MysqlStore) initialize(dsn string, upsertQuery string, enableUpsert
store.SupportBucketTable = false
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = DefaultUpsertQuery
}
store.SqlGenerator = &SqlGenMysql{
CreateTableSqlTemplate: "",
-5
View File
@@ -35,9 +35,6 @@ func (store *MysqlStore2) GetName() string {
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs avoid the duplicate-key roundtrip the
// inode-index KvPut would otherwise emit on every write.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"upsertQuery"),
@@ -60,8 +57,6 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
store.SupportBucketTable = true
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = mysql.DefaultUpsertQuery
}
store.SqlGenerator = &mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable,
-5
View File
@@ -13,11 +13,6 @@ type SqlGenPostgres struct {
UpsertQueryTemplate string
}
// DefaultUpsertQuery keeps INSERTs idempotent so a duplicate-key failure
// (23505) cannot poison the surrounding transaction (25P02). Used when the
// user enables upsert but does not provide their own template.
const DefaultUpsertQuery = `INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash, name) DO UPDATE SET directory=EXCLUDED.directory, meta=EXCLUDED.meta`
var (
_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
)
@@ -1,25 +0,0 @@
package postgres
import (
"strings"
"testing"
)
func TestDefaultUpsertQueryIsConflictSafe(t *testing.T) {
gen := &SqlGenPostgres{UpsertQueryTemplate: DefaultUpsertQuery}
got := gen.GetSqlInsert("filemeta")
if !strings.Contains(got, "ON CONFLICT") {
t.Fatalf("expected ON CONFLICT in default upsert, got: %s", got)
}
if !strings.Contains(got, `"filemeta"`) {
t.Fatalf("expected quoted table name, got: %s", got)
}
}
func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
gen := &SqlGenPostgres{}
got := gen.GetSqlInsert("filemeta")
if strings.Contains(got, "ON CONFLICT") {
t.Fatalf("plain INSERT path should not contain ON CONFLICT, got: %s", got)
}
}
-5
View File
@@ -30,9 +30,6 @@ func (store *PostgresStore) GetName() string {
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs are not exposed to duplicate-key tx
// poisoning on Postgres; an explicit false still disables it.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
@@ -59,8 +56,6 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
store.SupportBucketTable = false
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = DefaultUpsertQuery
}
store.SqlGenerator = &SqlGenPostgres{
CreateTableSqlTemplate: "",
-5
View File
@@ -35,9 +35,6 @@ func (store *PostgresStore2) GetName() string {
func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs are not exposed to duplicate-key tx
// poisoning on Postgres; an explicit false still disables it.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"upsertQuery"),
@@ -65,8 +62,6 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
store.SupportBucketTable = true
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = postgres.DefaultUpsertQuery
}
store.SqlGenerator = &postgres.SqlGenPostgres{
CreateTableSqlTemplate: createTable,
+1 -13
View File
@@ -124,23 +124,11 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) {
// apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST"))
apiRouter.Methods(http.MethodPost).Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN))
// Health probes
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(iama.healthzHandler)
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(iama.readyzHandler)
//
// NotFound
apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
}
func (iama *IamApiServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (iama *IamApiServer) readyzHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
// Shutdown gracefully stops the IAM API server and releases resources.
// It cancels the master client connection goroutine and closes gRPC connections.
// This method is safe to call multiple times.
+1 -2
View File
@@ -1,7 +1,6 @@
package mount
import (
"context"
"sync/atomic"
"google.golang.org/grpc"
@@ -22,7 +21,7 @@ func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFile
for x := 0; x < n; x++ {
filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress()
err = pb.WithGrpcClient(context.Background(), streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
err = pb.WithGrpcClient(streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, filerGrpcAddress, false, wfs.option.GrpcDialOption)
+2 -5
View File
@@ -91,10 +91,7 @@ func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialO
// Per-attempt timeout to prevent a single slow RPC from consuming the entire retry budget
attemptCtx, attemptCancel := context.WithTimeout(deadlineCtx, 10*time.Second)
defer attemptCancel()
// Pass attemptCtx so its expiry is not mistaken for a dead shared
// connection: invalidating it would cancel every other in-flight
// assign with "the client connection is closing".
return WithMasterServerClient(attemptCtx, false, masterFn(attemptCtx), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
return WithMasterServerClient(false, masterFn(attemptCtx), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
req := &master_pb.AssignRequest{
Count: request.Count,
Replication: request.Replication,
@@ -154,7 +151,7 @@ func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialO
func LookupJwt(master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) {
WithMasterServerClient(context.Background(), false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
WithMasterServerClient(false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
resp, grpcErr := masterClient.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{
VolumeOrFileIds: []string{fileId},

Some files were not shown because too many files have changed in this diff Show More