mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-14 02:25:50 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a4bf9fe47f | ||
|
|
b3ea57b5d7 |
@@ -3,11 +3,6 @@ name: "go: build dev binaries"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/binaries_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
@@ -48,7 +43,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
@@ -97,7 +92,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -2,11 +2,6 @@ name: "Code Scanning - Action"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/codeql.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/codeql
|
||||
@@ -23,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -3,12 +3,6 @@ name: "docker: build dev containers"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/container_dev.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -28,7 +22,7 @@ jobs:
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
@@ -51,7 +45,7 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -81,7 +75,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
@@ -117,7 +111,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.1.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -126,7 +126,7 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.1.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Free Disk Space
|
||||
@@ -207,7 +207,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v4.1.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
@@ -282,7 +282,7 @@ jobs:
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
@@ -408,7 +408,7 @@ jobs:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.1.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
@@ -33,7 +33,6 @@ permissions:
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
IMAGE: ghcr.io/chrislusf/seaweedfs
|
||||
|
||||
# Limit concurrent builds to avoid rate limits
|
||||
concurrency:
|
||||
@@ -57,7 +56,7 @@ jobs:
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
@@ -80,7 +79,7 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -114,45 +113,59 @@ jobs:
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
# One job per (variant, platform) on a native runner, pushed by digest;
|
||||
# the merge job stitches the digests into one multi-arch tag.
|
||||
# ── Build Docker containers ─────────────────────────────────────────
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# Build sequentially to avoid rate limits
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
include:
|
||||
# Normal volume - multi-arch
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- variant: normal
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk - multi-arch
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
rust_variant: large-disk
|
||||
|
||||
# Full tags - multi-arch
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- variant: large_disk_full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
rust_variant: large-disk
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
rust_variant: large-disk
|
||||
|
||||
steps:
|
||||
- name: Skip unselected variant
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant
|
||||
run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0
|
||||
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
@@ -178,31 +191,36 @@ jobs:
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
sudo docker system prune -af --volumes
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
|
||||
|
||||
- name: Docker meta
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.IMAGE }}
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
|
||||
- name: Set up QEMU
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu
|
||||
uses: docker/setup-qemu-action@v4.1.0
|
||||
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -210,111 +228,91 @@ jobs:
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build and push ${{ matrix.variant }} (${{ matrix.platform }})
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
platforms: ${{ matrix.platforms }}
|
||||
# Push to GHCR to avoid Docker Hub rate limits on pulls
|
||||
tags: |
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
# Flat single-platform manifest so imagetools create assembles cleanly.
|
||||
provenance: false
|
||||
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
# max only for rocksdb: its RocksDB compile is sha-independent and worth
|
||||
# keeping; go-build layers are sha-busted every release, so min elsewhere.
|
||||
cache-to: type=gha,mode=${{ matrix.variant == 'rocksdb' && 'max' || 'min' }},scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
|
||||
build-args: |
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
|
||||
|
||||
- name: Export digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
|
||||
- name: Clean up build artifacts
|
||||
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
sudo docker system prune -f
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
- name: Upload digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: digest-${{ matrix.variant }}-${{ matrix.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Assemble each variant's per-platform digests into one tag, then mirror to Docker Hub.
|
||||
merge:
|
||||
needs: [build]
|
||||
copy-to-dockerhub:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant: [normal, large_disk, full, large_disk_full, rocksdb]
|
||||
include:
|
||||
- { variant: normal, tag_suffix: "" }
|
||||
- { variant: large_disk, tag_suffix: _large_disk }
|
||||
- { variant: full, tag_suffix: _full }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full }
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb }
|
||||
- variant: normal
|
||||
tag_suffix: ""
|
||||
- variant: large_disk
|
||||
tag_suffix: _large_disk
|
||||
- variant: full
|
||||
tag_suffix: _full
|
||||
- variant: large_disk_full
|
||||
tag_suffix: _large_disk_full
|
||||
- variant: rocksdb
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
|
||||
steps:
|
||||
- name: Download digests
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: digest-${{ matrix.variant }}-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Create multi-arch tag on GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
|
||||
docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -322,37 +320,45 @@ jobs:
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Copy ${{ matrix.variant }} to Docker Hub
|
||||
|
||||
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
# Function to retry with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
|
||||
# Copy multi-arch image from GHCR to Docker Hub with retry
|
||||
# This is much more efficient than pulling/pushing individual arch images
|
||||
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
|
||||
retry_with_backoff crane copy \
|
||||
${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
echo "Copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
@@ -361,7 +367,7 @@ jobs:
|
||||
# CVE, base-image lag).
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
needs: [build]
|
||||
if: github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
@@ -388,7 +394,7 @@ jobs:
|
||||
# Scan the multi-arch tag on GHCR (already pushed by the build job).
|
||||
# Trivy scans the runner's native platform; OS packages are identical
|
||||
# across architectures since they all share the same alpine base.
|
||||
image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
@@ -412,7 +418,7 @@ jobs:
|
||||
# trivy-scan: vuln findings are reported but do not block `latest`.
|
||||
tag-latest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -469,24 +475,24 @@ jobs:
|
||||
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
|
||||
DST_TAG="${{ matrix.latest_tag }}"
|
||||
|
||||
echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}"
|
||||
echo "Tagging ghcr.io/chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "ghcr.io/chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v2
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v1
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "End to End"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/e2e
|
||||
@@ -36,23 +24,18 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "EC Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,7 +22,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -3,18 +3,8 @@ name: "go: build binary"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/go
|
||||
@@ -30,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
@@ -47,27 +37,12 @@ jobs:
|
||||
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
|
||||
if grep -q "vet:" vet-output.txt; then exit 1; fi
|
||||
|
||||
vet-32bit:
|
||||
name: Go Vet 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
|
||||
run: |
|
||||
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
|
||||
if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
@@ -80,26 +55,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Test
|
||||
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
|
||||
|
||||
test-32bit:
|
||||
name: Test 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
# 386 test binaries run natively on the amd64 runner. This catches what vet
|
||||
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
- name: Test linux/386
|
||||
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
|
||||
|
||||
+24
-115
@@ -16,7 +16,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -51,71 +51,22 @@ jobs:
|
||||
|
||||
echo "=== Testing default configuration ==="
|
||||
helm template test $CHART_DIR > /tmp/default.yaml
|
||||
echo "Default configuration renders successfully"
|
||||
echo "✓ Default configuration renders successfully"
|
||||
|
||||
echo "=== Testing with S3 enabled ==="
|
||||
helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
|
||||
grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
|
||||
echo "S3 deployment renders correctly"
|
||||
echo "✓ S3 deployment renders correctly"
|
||||
|
||||
echo "=== Testing with all-in-one mode ==="
|
||||
helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
|
||||
grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
|
||||
echo "All-in-one deployment renders correctly"
|
||||
echo "✓ All-in-one deployment renders correctly"
|
||||
|
||||
echo "=== Testing with security enabled ==="
|
||||
helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true > /tmp/security.yaml
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "Security configuration renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing JWT expiration overrides ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
|
||||
> /tmp/jwt-volume-write-expiration.yaml
|
||||
grep -q "security-config" /tmp/jwt-volume-write-expiration.yaml
|
||||
grep -q "expires_after_seconds = 11" /tmp/jwt-volume-write-expiration.yaml
|
||||
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeRead=22 \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerWrite=33 \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerRead=44 \
|
||||
> /tmp/jwt-expiration.yaml
|
||||
|
||||
assert_jwt_expiration() {
|
||||
local section="$1"
|
||||
local seconds="$2"
|
||||
awk -v section="[$section]" -v seconds="$seconds" '
|
||||
/^[[:space:]]*\[.*\][[:space:]]*$/ {
|
||||
in_section = index($0, section) > 0
|
||||
}
|
||||
in_section && $0 ~ "^[[:space:]]*expires_after_seconds = " seconds "$" {
|
||||
found = 1
|
||||
}
|
||||
END { exit !found }
|
||||
' /tmp/jwt-expiration.yaml
|
||||
}
|
||||
|
||||
assert_jwt_expiration jwt.signing 11
|
||||
assert_jwt_expiration jwt.signing.read 22
|
||||
assert_jwt_expiration jwt.filer_signing 33
|
||||
assert_jwt_expiration jwt.filer_signing.read 44
|
||||
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.enableSecurity=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
|
||||
> /tmp/jwt-default-expiration.yaml
|
||||
if grep -q "expires_after_seconds =" /tmp/jwt-default-expiration.yaml; then
|
||||
echo "FAIL: zero JWT expiration values should preserve runtime defaults"
|
||||
exit 1
|
||||
fi
|
||||
echo "JWT expiration overrides render correctly"
|
||||
echo "✓ Security configuration renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing IAM gRPC opt-in path ==="
|
||||
@@ -170,7 +121,7 @@ jobs:
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append("defaults: security ConfigMap should not render")
|
||||
else:
|
||||
print("defaults: no security-config ConfigMap (unchanged)")
|
||||
print("✓ defaults: no security-config ConfigMap (unchanged)")
|
||||
|
||||
# Case 2: filerWrite=true alone is the documented opt-in for
|
||||
# the Admin UI Users tab. Configmap must render with
|
||||
@@ -190,7 +141,7 @@ jobs:
|
||||
if "[grpc" in toml:
|
||||
failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
|
||||
if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
|
||||
print("filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
|
||||
print("✓ filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
|
||||
|
||||
# Case 3: filer + admin pods must MOUNT the security ConfigMap
|
||||
# under filerWrite=true so the JWT key reaches both processes.
|
||||
@@ -203,7 +154,7 @@ jobs:
|
||||
if "security-config" not in vols or "security-config" not in mounts:
|
||||
failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
|
||||
else:
|
||||
print(f"filerWrite=true: {wl} mounts security-config")
|
||||
print(f"✓ filerWrite=true: {wl} mounts security-config")
|
||||
cert_vols = {v for v in vols if v.endswith("-cert")}
|
||||
if cert_vols:
|
||||
failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")
|
||||
@@ -221,7 +172,7 @@ jobs:
|
||||
if missing:
|
||||
failed.append(f"enableSecurity=true: security.toml missing {missing}")
|
||||
else:
|
||||
print("enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
|
||||
print("✓ enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
|
||||
|
||||
# Case 5: helper must tolerate explicit nulls (gemini-code-assist
|
||||
# PR review). securityConfig=null was the parens-pattern crash
|
||||
@@ -236,7 +187,7 @@ jobs:
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append(f"{null_path}=null: should not render configmap")
|
||||
else:
|
||||
print(f"{null_path}=null: render tolerates explicit null")
|
||||
print(f"✓ {null_path}=null: render tolerates explicit null")
|
||||
|
||||
if failed:
|
||||
print("\nFAIL:", file=sys.stderr)
|
||||
@@ -244,14 +195,14 @@ jobs:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
echo "IAM gRPC decoupling tests passed"
|
||||
echo "✓ IAM gRPC decoupling tests passed"
|
||||
|
||||
echo "=== Testing with monitoring enabled ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.monitoring.enabled=true \
|
||||
--set global.seaweedfs.monitoring.gatewayHost=prometheus \
|
||||
--set global.seaweedfs.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
|
||||
echo "Monitoring configuration renders correctly"
|
||||
echo "✓ Monitoring configuration renders correctly"
|
||||
|
||||
echo "=== Testing with PVC storage ==="
|
||||
helm template test $CHART_DIR \
|
||||
@@ -259,25 +210,25 @@ jobs:
|
||||
--set master.data.size=10Gi \
|
||||
--set master.data.storageClass=standard > /tmp/pvc.yaml
|
||||
grep -q "PersistentVolumeClaim" /tmp/pvc.yaml
|
||||
echo "PVC configuration renders correctly"
|
||||
echo "✓ PVC configuration renders correctly"
|
||||
|
||||
echo "=== Testing with custom replicas ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set master.replicas=3 \
|
||||
--set filer.replicas=2 \
|
||||
--set volume.replicas=3 > /tmp/replicas.yaml
|
||||
echo "Custom replicas configuration renders correctly"
|
||||
echo "✓ Custom replicas configuration renders correctly"
|
||||
|
||||
echo "=== Testing filer with S3 gateway ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set filer.s3.enabled=true \
|
||||
--set filer.s3.enableAuth=true > /tmp/filer-s3.yaml
|
||||
echo "Filer S3 gateway renders correctly"
|
||||
echo "✓ Filer S3 gateway renders correctly"
|
||||
|
||||
echo "=== Testing SFTP enabled ==="
|
||||
helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml
|
||||
grep -q "seaweedfs-sftp" /tmp/sftp.yaml
|
||||
echo "SFTP deployment renders correctly"
|
||||
echo "✓ SFTP deployment renders correctly"
|
||||
|
||||
echo "=== Testing ingress configurations ==="
|
||||
helm template test $CHART_DIR \
|
||||
@@ -286,12 +237,12 @@ jobs:
|
||||
--set s3.enabled=true \
|
||||
--set s3.ingress.enabled=true > /tmp/ingress.yaml
|
||||
grep -q "kind: Ingress" /tmp/ingress.yaml
|
||||
echo "Ingress configurations render correctly"
|
||||
echo "✓ Ingress configurations render correctly"
|
||||
|
||||
echo "=== Testing COSI driver ==="
|
||||
helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
|
||||
grep -q "seaweedfs-cosi" /tmp/cosi.yaml
|
||||
echo "COSI driver renders correctly"
|
||||
echo "✓ COSI driver renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing long release name: service names match DNS references ==="
|
||||
@@ -334,7 +285,7 @@ jobs:
|
||||
[ "$MASTER_SVC" = "$MASTER_ADDR_SVC" ] || { echo "FAIL: master service name mismatch"; exit 1; }
|
||||
[ "$FILER_CLIENT_SVC" = "$FILER_ADDR_SVC" ] || { echo "FAIL: filer-client service name mismatch"; exit 1; }
|
||||
[ "$FILER_CLIENT_SVC" = "$S3_FILER_SVC" ] || { echo "FAIL: S3 -filer= does not match filer-client service"; exit 1; }
|
||||
echo "Normal mode: service names match DNS references with long release name"
|
||||
echo "✓ Normal mode: service names match DNS references with long release name"
|
||||
|
||||
# --- All-in-one mode: all-in-one service vs both helper addresses ---
|
||||
helm template "$LONG_RELEASE" $CHART_DIR \
|
||||
@@ -354,7 +305,7 @@ jobs:
|
||||
|
||||
[ "$AIO_SVC" = "$AIO_MASTER_ADDR_SVC" ] || { echo "FAIL: all-in-one master address mismatch"; exit 1; }
|
||||
[ "$AIO_SVC" = "$AIO_FILER_ADDR_SVC" ] || { echo "FAIL: all-in-one filer address mismatch"; exit 1; }
|
||||
echo "All-in-one mode: service names match DNS references with long release name"
|
||||
echo "✓ All-in-one mode: service names match DNS references with long release name"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing security+S3: no blank lines in shell command blocks ==="
|
||||
@@ -392,7 +343,7 @@ jobs:
|
||||
print(f"FAIL: {e}", file=sys.stderr)
|
||||
print("Rendered with: global.seaweedfs.enableSecurity=true, filer.s3.enabled=true, s3.enabled=true, allInOne.enabled=true", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("No blank lines in security+S3 command blocks")
|
||||
print("✓ No blank lines in security+S3 command blocks")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
@@ -470,7 +421,7 @@ jobs:
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"{label}: cert/key/https args emitted together")
|
||||
print(f"✓ {label}: cert/key/https args emitted together")
|
||||
else:
|
||||
if has_cert or has_key or has_https:
|
||||
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
|
||||
@@ -478,7 +429,7 @@ jobs:
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"{label}: no TLS args emitted, main -port stays HTTP")
|
||||
print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
|
||||
|
||||
# bash -n: pin down that the rendered script parses. Guards against
|
||||
# a future helper change that leaves a dangling `\` with nothing
|
||||
@@ -495,49 +446,7 @@ jobs:
|
||||
sys.exit(1 if failed else 0)
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== Testing all-in-one env: a key in both global and component renders once ==="
|
||||
# Regression: all-in-one looped global and component extraEnvironmentVars
|
||||
# in two separate ranges, emitting duplicate env entries for any key set
|
||||
# in both maps. Render a shared key and assert it appears exactly once in
|
||||
# the all-in-one container, with the component value winning (consistent
|
||||
# with the merge helper the other components already use). pyyaml is
|
||||
# installed by the earlier IAM gRPC block in this same step.
|
||||
helm template test $CHART_DIR \
|
||||
--set allInOne.enabled=true \
|
||||
--set global.seaweedfs.extraEnvironmentVars.WEED_SHARED=fromGlobal \
|
||||
--set allInOne.extraEnvironmentVars.WEED_SHARED=fromComponent > /tmp/aio-env.yaml
|
||||
python3 - /tmp/aio-env.yaml <<'PYEOF'
|
||||
import sys, yaml
|
||||
from collections import Counter
|
||||
docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if d]
|
||||
dep = next(d for d in docs if d.get("kind") == "Deployment"
|
||||
and d["metadata"]["name"].endswith("all-in-one"))
|
||||
envs = [e["name"] for c in dep["spec"]["template"]["spec"]["containers"]
|
||||
for e in c.get("env", [])]
|
||||
dups = {k: v for k, v in Counter(envs).items() if v > 1}
|
||||
if dups:
|
||||
print(f"FAIL: duplicate env entries in all-in-one container: {dups}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
val = next(e.get("value") for c in dep["spec"]["template"]["spec"]["containers"]
|
||||
for e in c.get("env", []) if e["name"] == "WEED_SHARED")
|
||||
if val != "fromComponent":
|
||||
print(f"FAIL: WEED_SHARED should take the component value 'fromComponent', got '{val}'",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("all-in-one env: shared key renders once, component value wins")
|
||||
PYEOF
|
||||
|
||||
echo "=== Testing bucket versioning: YAML bool false suspends like string \"false\" ==="
|
||||
# bool false used to be a silent no-op while string "false" suspended.
|
||||
BOOL_FALSE=$(helm template test $CHART_DIR \
|
||||
--set s3.enabled=true \
|
||||
--set s3.createBuckets[0].name=verbucket \
|
||||
--set s3.createBuckets[0].versioning=false | grep 's3.bucket.versioning -name verbucket' || true)
|
||||
echo "$BOOL_FALSE" | grep -q -- '-status Suspended' || { echo "FAIL: bool false versioning did not Suspend the bucket"; exit 1; }
|
||||
echo "Bucket versioning: YAML bool false suspends consistently with string \"false\""
|
||||
|
||||
echo "All template rendering tests passed!"
|
||||
echo "✅ All template rendering tests passed!"
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.14.0
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v5
|
||||
|
||||
@@ -3,24 +3,8 @@ name: "Kafka Quick Test (Load Test with Schema Registry)"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
concurrency:
|
||||
@@ -37,7 +21,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -48,11 +32,6 @@ jobs:
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
|
||||
@@ -3,24 +3,8 @@ name: "Kafka Gateway Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/kafka-tests
|
||||
@@ -57,7 +41,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Container Environment
|
||||
run: |
|
||||
@@ -102,7 +86,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Integration Container Environment
|
||||
run: |
|
||||
@@ -143,7 +127,7 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -322,7 +306,7 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -484,7 +468,7 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -642,7 +626,7 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -809,7 +793,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Protocol Container Environment
|
||||
run: |
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -40,7 +40,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
name: "NFS Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/nfs-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
nfs-integration:
|
||||
name: NFS Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run NFS Integration Tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running NFS integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run the protocol-layer tests. The kernel-mount tests require root
|
||||
# for mount(2) and are exercised in their own privileged step below;
|
||||
# skip them here so a "skipped because not root" line doesn't show
|
||||
# up as noise on every CI run.
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "NFS integration tests completed"
|
||||
|
||||
- name: Install kernel NFS client
|
||||
run: |
|
||||
# nfs-common provides mount.nfs; netbase provides /etc/protocols
|
||||
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nfs-common netbase
|
||||
|
||||
- name: Run kernel-mount E2E tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running kernel-mount end-to-end tests..."
|
||||
echo "These mount the running 'weed nfs' subprocess via the actual"
|
||||
echo "Linux NFS client to catch protocol regressions invisible to"
|
||||
echo "the go-nfs-client-based tests above."
|
||||
echo "============================================"
|
||||
|
||||
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
|
||||
# binary that test/nfs/framework.go locates via $PATH) resolve
|
||||
# correctly under sudo, and pass through the Go module/cache dirs
|
||||
# so we don't redownload modules under root.
|
||||
sudo env "PATH=$PATH" \
|
||||
GOMODCACHE="$(go env GOMODCACHE)" \
|
||||
GOCACHE="$(go env GOCACHE)" \
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Kernel-mount E2E tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
|
||||
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
|
||||
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
|
||||
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
|
||||
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1,477 +0,0 @@
|
||||
name: "Performance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- 'seaweed-volume/**'
|
||||
- 'test/perf/**'
|
||||
- '.github/workflows/performance.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
profile_duration:
|
||||
description: "CPU profiling duration in seconds"
|
||||
required: false
|
||||
default: "30"
|
||||
type: string
|
||||
benchmark_files:
|
||||
description: "Number of files for the throughput benchmark"
|
||||
required: false
|
||||
default: "100000"
|
||||
type: string
|
||||
benchmark_concurrency:
|
||||
description: "Concurrent read/write workers"
|
||||
required: false
|
||||
default: "16"
|
||||
type: string
|
||||
benchmark_size:
|
||||
description: "Simulated file size in bytes"
|
||||
required: false
|
||||
default: "1024"
|
||||
type: string
|
||||
s3_objects:
|
||||
description: "Number of objects for the S3 benchmark"
|
||||
required: false
|
||||
default: "20000"
|
||||
type: string
|
||||
s3_size:
|
||||
description: "S3 object size in bytes"
|
||||
required: false
|
||||
default: "4096"
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/performance
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
VOL_SIZE_LIMIT: "1024"
|
||||
|
||||
jobs:
|
||||
performance-profile:
|
||||
name: CPU and Heap Profile
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Start server with profiling enabled
|
||||
run: |
|
||||
mkdir -p ./perfdata
|
||||
./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \
|
||||
-s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \
|
||||
-s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> weed.log 2>&1 &
|
||||
echo "WEED_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# give the volume server a moment to register with the master
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Capture profiles under load
|
||||
run: |
|
||||
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
|
||||
# drive write load so the sampled profile reflects real work
|
||||
./weed_bin benchmark -master=localhost:9333 -writeOnly \
|
||||
-c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 &
|
||||
echo "Sampling CPU profile for ${DURATION}s..."
|
||||
curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt
|
||||
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
|
||||
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS)"
|
||||
echo '```'
|
||||
if [ -f mem-profile.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Profile summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## CPU profile (top functions)"
|
||||
echo '```'
|
||||
head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop server
|
||||
if: always()
|
||||
run: kill "${WEED_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: tail -200 weed.log || true
|
||||
|
||||
- name: Upload profiles
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-profile-${{ github.run_number }}
|
||||
path: |
|
||||
cpu.pprof
|
||||
heap.pprof
|
||||
cpu-top.txt
|
||||
heap-top.txt
|
||||
goroutine.txt
|
||||
mem-profile.csv
|
||||
mem-profile.csv.peak
|
||||
weed.log
|
||||
retention-days: 30
|
||||
|
||||
benchmark:
|
||||
name: Throughput Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
if: matrix.impl == 'rust'
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start master and volume server
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# let the volume server register with the master via heartbeat
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-benchmark.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run throughput benchmark
|
||||
run: |
|
||||
N="${{ github.event.inputs.benchmark_files || '100000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.benchmark_size || '1024' }}"
|
||||
./weed_bin benchmark -master=localhost:9333 \
|
||||
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
|
||||
|
||||
- name: Run Go micro-benchmarks
|
||||
if: matrix.impl == 'go'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
go test -run='^$' -bench=. -benchmem -benchtime=10x \
|
||||
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
|
||||
2>&1 | tee go-benchmarks.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-benchmark.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Throughput benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
|
||||
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
|
||||
- name: Upload benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
benchmark-results.txt
|
||||
go-benchmarks.txt
|
||||
mem-benchmark.csv
|
||||
mem-benchmark.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
retention-days: 7
|
||||
|
||||
s3-benchmark:
|
||||
name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
if: matrix.impl == 'rust'
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed and S3 load tool
|
||||
run: |
|
||||
go build -o weed_bin ./weed
|
||||
go build -o s3bench ./test/s3/benchmark
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start cluster with S3 gateway
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 3
|
||||
./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
|
||||
-s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> filer.log 2>&1 &
|
||||
echo "FILER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z localhost 8000 2>/dev/null; then
|
||||
echo "s3 gateway is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 2
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-s3.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run S3 read/write benchmark
|
||||
run: |
|
||||
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.s3_size || '4096' }}"
|
||||
./s3bench -endpoint=http://localhost:8000 \
|
||||
-access-key=some_access_key1 -secret-key=some_secret_key1 \
|
||||
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
|
||||
2>&1 | tee s3-benchmark-results.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-s3.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: S3 benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## S3 read/write benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
|
||||
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
|
||||
|
||||
- name: Upload S3 benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
s3-benchmark-results.txt
|
||||
mem-s3.csv
|
||||
mem-s3.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
filer.log
|
||||
retention-days: 7
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "Plugin Worker Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -45,7 +33,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Run plugin worker tests
|
||||
run: go test -v ./${{ matrix.path }}
|
||||
|
||||
@@ -3,24 +3,8 @@ name: "PostgreSQL Gateway Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/postgres-tests
|
||||
@@ -45,18 +29,13 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
name: "release: bump version"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
bump:
|
||||
description: "Which part to increment (ignored when 'version' is set)"
|
||||
type: choice
|
||||
options:
|
||||
- minor
|
||||
- major
|
||||
default: minor
|
||||
version:
|
||||
description: "Explicit MAJOR.MINOR to set, e.g. 4.36 (overrides 'bump')"
|
||||
type: string
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
bump-version:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Compute new version
|
||||
id: compute
|
||||
env:
|
||||
BUMP: ${{ inputs.bump }}
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
|
||||
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
echo "current: ${MAJOR}.${MINOR}"
|
||||
|
||||
if [ -n "$INPUT_VERSION" ]; then
|
||||
if ! [[ "$INPUT_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR, e.g. 5.01 (got '$INPUT_VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
# 10# forces base 10 so 08/09 are not parsed as octal.
|
||||
MAJOR=$((10#${INPUT_VERSION%%.*}))
|
||||
MINOR=$((10#${INPUT_VERSION##*.}))
|
||||
if [ "$MINOR" -gt 99 ]; then
|
||||
echo "::error::minor must be 0-99 (got $MINOR); it rolls into the next major at 99"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
case "$BUMP" in
|
||||
# Minor is a 2-digit field: 4.99 -> 5.00 -> 5.01.
|
||||
major) MAJOR=$((MAJOR + 1)); MINOR=0 ;;
|
||||
minor)
|
||||
if [ "$MINOR" -ge 99 ]; then
|
||||
MAJOR=$((MAJOR + 1)); MINOR=0
|
||||
else
|
||||
MINOR=$((MINOR + 1))
|
||||
fi
|
||||
;;
|
||||
*) echo "::error::unknown bump '$BUMP'"; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# appVersion mirrors the Go VERSION_NUMBER (zero-padded minor);
|
||||
# chart version is plain SemVer (no leading zeros).
|
||||
APP_VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
|
||||
CHART_VERSION="${MAJOR}.${MINOR}.0"
|
||||
echo "new: app=${APP_VERSION} chart=${CHART_VERSION}"
|
||||
|
||||
{
|
||||
echo "major=${MAJOR}"
|
||||
echo "minor=${MINOR}"
|
||||
echo "app_version=${APP_VERSION}"
|
||||
echo "chart_version=${CHART_VERSION}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Apply version to constants.go
|
||||
env:
|
||||
MAJOR: ${{ steps.compute.outputs.major }}
|
||||
MINOR: ${{ steps.compute.outputs.minor }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
sed -i -E "s/(MAJOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MAJOR}\2/" "$CONST"
|
||||
sed -i -E "s/(MINOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MINOR}\2/" "$CONST"
|
||||
grep -E 'MAJOR_VERSION|MINOR_VERSION' "$CONST"
|
||||
|
||||
- name: Apply version to Chart.yaml
|
||||
env:
|
||||
APP_VERSION: ${{ steps.compute.outputs.app_version }}
|
||||
CHART_VERSION: ${{ steps.compute.outputs.chart_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHART=k8s/charts/seaweedfs/Chart.yaml
|
||||
sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "$CHART"
|
||||
sed -i -E "s/^version:.*/version: ${CHART_VERSION}/" "$CHART"
|
||||
cat "$CHART"
|
||||
|
||||
- name: Commit and push
|
||||
env:
|
||||
APP_VERSION: ${{ steps.compute.outputs.app_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet; then
|
||||
echo "No version change to commit."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add weed/util/version/constants.go k8s/charts/seaweedfs/Chart.yaml
|
||||
git commit -m "${APP_VERSION}"
|
||||
git push
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -142,7 +142,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -156,7 +156,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
asset_suffix: linux-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
asset_suffix: darwin-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
@@ -119,7 +119,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
cross: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -66,42 +66,34 @@ jobs:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
# Large disk (default, 5bytes feature)
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
# Normal volume size
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -112,9 +104,7 @@ jobs:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
@@ -129,7 +119,7 @@ jobs:
|
||||
asset_suffix: darwin_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
@@ -140,7 +130,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -155,40 +145,32 @@ jobs:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5 -r "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -199,9 +181,7 @@ jobs:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
@@ -209,7 +189,7 @@ jobs:
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: choco install protoc -y
|
||||
@@ -218,7 +198,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -233,42 +213,33 @@ jobs:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target-dir target/large-disk
|
||||
cargo build --release
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --no-default-features --target-dir target/normal
|
||||
cargo build --release --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
shell: bash
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
|
||||
rm weed-volume-large-disk.exe
|
||||
|
||||
cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
|
||||
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Generate md5 checksums
|
||||
shell: bash
|
||||
run: |
|
||||
for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -279,6 +250,4 @@ jobs:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -2,16 +2,7 @@ name: "S3 Authenticated Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/normal/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-example-integration-tests.yml'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-integration-tests
|
||||
cancel-in-progress: true
|
||||
@@ -27,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -2,15 +2,7 @@ name: "S3 Filer Group Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/filer_group/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-filer-group-tests.yml'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
@@ -30,7 +22,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -2,15 +2,7 @@ name: "S3 Go Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-go-tests.yml'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-go-tests
|
||||
cancel-in-progress: true
|
||||
@@ -33,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -97,7 +89,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -145,7 +137,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -196,7 +188,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -286,7 +278,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -331,7 +323,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -389,7 +381,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -453,7 +445,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -502,7 +494,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -570,7 +562,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -625,7 +617,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -262,7 +262,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -313,7 +313,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -400,7 +400,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -3,22 +3,8 @@ name: "S3 Proxy Signature Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
|
||||
@@ -34,7 +20,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -42,11 +28,6 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -33,19 +33,15 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
run: docker pull apache/spark:3.5.8
|
||||
|
||||
- name: Run S3 Spark integration tests
|
||||
working-directory: test/s3/spark
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -209,7 +209,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -258,7 +258,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -357,7 +357,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -2,14 +2,6 @@ name: "S3 Tables Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3tables/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-tables-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -23,7 +15,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -83,7 +75,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -91,11 +83,6 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
@@ -144,7 +131,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -152,15 +139,11 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Trino image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull trinodb/trino:479
|
||||
run: docker pull trinodb/trino:479
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -214,7 +197,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -222,16 +205,14 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull dremio/dremio-oss:25.2.0
|
||||
pull python:3.11-slim
|
||||
- name: Pre-pull Dremio image
|
||||
run: docker pull dremio/dremio-oss:25.2.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -287,7 +268,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -295,16 +276,14 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/doris:doris-all-in-one-2.1.0
|
||||
pull python:3.11-slim
|
||||
- name: Pre-pull Doris image
|
||||
run: docker pull apache/doris:doris-all-in-one-2.1.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -360,7 +339,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -375,15 +354,8 @@ jobs:
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Pre-pull Polaris image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/polaris:latest
|
||||
run: docker pull apache/polaris:latest
|
||||
|
||||
- name: Run Polaris Integration Tests
|
||||
timeout-minutes: 25
|
||||
@@ -428,7 +400,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -436,15 +408,11 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
run: docker pull apache/spark:3.5.1
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -498,7 +466,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -506,16 +474,13 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull RisingWave image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull risingwavelabs/risingwave:v2.5.0
|
||||
pull postgres:16-alpine
|
||||
docker pull risingwavelabs/risingwave:v2.5.0
|
||||
docker pull postgres:16-alpine
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -569,7 +534,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -577,39 +542,11 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Week stamp for image cache key
|
||||
id: week
|
||||
run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore python:3 image cache
|
||||
id: python-image
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/python3-image.tar
|
||||
key: python3-image-${{ steps.week.outputs.week }}
|
||||
restore-keys: |
|
||||
python3-image-
|
||||
|
||||
- name: Load or pull python:3
|
||||
run: |
|
||||
if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then
|
||||
docker load -i /tmp/python3-image.tar
|
||||
exit 0
|
||||
fi
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
if pull python:3; then
|
||||
docker save -o /tmp/python3-image.tar python:3
|
||||
elif [ -f /tmp/python3-image.tar ]; then
|
||||
# Docker Hub unreachable; fall back to last week's cached image
|
||||
docker load -i /tmp/python3-image.tar
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -663,7 +600,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -671,6 +608,15 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
|
||||
- name: Pre-pull LocalStack image (if needed)
|
||||
run: docker pull localstack/localstack:latest || true
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
@@ -723,7 +669,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -731,16 +677,14 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull unitycatalog/unitycatalog:v0.4.0
|
||||
pull python:3.11-slim
|
||||
- name: Pre-pull Unity Catalog image
|
||||
run: docker pull unitycatalog/unitycatalog:v0.4.0
|
||||
|
||||
- name: Pre-pull Python image for delta-rs writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -796,7 +740,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -861,7 +805,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
@@ -912,7 +856,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -3,26 +3,8 @@ name: "Ceph S3 tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/compatibility/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/compatibility/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3tests
|
||||
@@ -38,7 +20,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -63,7 +45,7 @@ jobs:
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
||||
python3 test/s3/fix_s3_tests_bucket_conflicts.py
|
||||
env:
|
||||
S3_TESTS_PATH: s3-tests
|
||||
|
||||
@@ -381,7 +363,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -406,7 +388,7 @@ jobs:
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
||||
python3 test/s3/fix_s3_tests_bucket_conflicts.py
|
||||
env:
|
||||
S3_TESTS_PATH: s3-tests
|
||||
|
||||
@@ -551,7 +533,7 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -676,7 +658,7 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
@@ -776,7 +758,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
# SETUP & BUILD
|
||||
# ========================================
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v5
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
- name: Cache Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
id: cache-spark
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: spark-3.5.0-bin-hadoop3
|
||||
key: spark-3.5.0-hadoop3
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
name: "terraform: validate and test modules"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: fmt, validate, plan-level tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: fmt check
|
||||
working-directory: terraform
|
||||
run: tofu fmt -recursive -check -diff
|
||||
|
||||
- name: validate core
|
||||
working-directory: terraform/modules/core
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: validate security
|
||||
working-directory: terraform/modules/security
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: plan-level tests (core)
|
||||
working-directory: terraform/modules/core
|
||||
run: tofu test
|
||||
|
||||
- name: validate examples
|
||||
run: |
|
||||
set -e
|
||||
for ex in terraform/examples/*/; do
|
||||
echo "== validate $ex =="
|
||||
tofu -chdir="$ex" init -backend=false -input=false
|
||||
tofu -chdir="$ex" validate
|
||||
done
|
||||
|
||||
smoke:
|
||||
name: local cluster smoke test (real weed)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o "$RUNNER_TEMP/weed" ./weed
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: Run local cluster harness
|
||||
working-directory: terraform/test/local
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
|
||||
|
||||
- name: Run local mTLS cluster harness
|
||||
working-directory: terraform/test/local-secure
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh
|
||||
@@ -3,20 +3,8 @@ name: "test s3 over https using aws-cli"
|
||||
on:
|
||||
push:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
pull_request:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
@@ -32,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "TLS Rotation Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,7 +22,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
|
||||
@@ -2,13 +2,6 @@ name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/tus/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
@@ -29,7 +22,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "Vacuum Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,7 +22,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
@@ -35,7 +35,6 @@ Your support will be really appreciated by me and other supporters!
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.piknik.com)
|
||||
[](https://www.keepsec.ca)
|
||||
[](https://zyner.org)
|
||||
|
||||
---
|
||||
|
||||
@@ -426,7 +425,7 @@ If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage
|
||||
|
||||
## SeaweedFS Filer ##
|
||||
|
||||
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory structure is an interface that is implemented in many key-value stores or databases.
|
||||
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory sturcture is an interface that is implemented in many key-value stores or databases.
|
||||
|
||||
The content of a file is mapped to one or many blobs, distributed to multiple volumes on multiple volume servers.
|
||||
|
||||
@@ -655,8 +654,7 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
|
||||
## Enterprise ##
|
||||
|
||||
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
|
||||
which has advanced features, including data recovery, self-healing storage,
|
||||
customizable erasure coding, EC vacuum and repair, etc.
|
||||
which has a self-healing storage format with better data protection.
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
|
||||
+15
-69
@@ -1,78 +1,24 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported versions
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Security fixes land in the latest release. Please reproduce against a recent
|
||||
release or `master` before reporting; issues that only reproduce on old,
|
||||
unsupported versions are not eligible for a fix or an advisory.
|
||||
If you find a security issue in SeaweedFS, please report it privately:
|
||||
|
||||
## Reporting a vulnerability
|
||||
- Email: support@seaweedfs.com
|
||||
- Do not open a public GitHub issue
|
||||
|
||||
Report privately through GitHub private vulnerability reporting (the "Report a
|
||||
vulnerability" button under the repository's Security tab). This keeps the
|
||||
report, the fix, and any CVE in one place. Do not open a public issue.
|
||||
Please include:
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce (if possible)
|
||||
- Affected versions
|
||||
|
||||
### What a report must include
|
||||
## Response
|
||||
|
||||
We can only act on reports that show real impact. Please include:
|
||||
- We will respond as soon as possible (usually within 1 business day)
|
||||
- We will investigate and work on a fix
|
||||
- We may coordinate disclosure with you
|
||||
|
||||
- Affected version (a release tag or `master` commit you reproduced on)
|
||||
- The exact deployment and configuration: which components are running
|
||||
(master, volume, filer, S3, admin), which ports are reachable by the
|
||||
attacker, and what authentication is enabled
|
||||
- The attacker's starting position: unauthenticated, a valid S3 user, an admin,
|
||||
or someone with access to the internal cluster network
|
||||
- The trust boundary that is crossed (e.g. an unauthenticated client reading
|
||||
another tenant's data, an S3 user escalating to admin)
|
||||
- A minimal, working reproduction or proof of concept
|
||||
- Expected vs. actual behavior
|
||||
## Notes
|
||||
|
||||
A report without a working reproduction and a clear trust boundary is a
|
||||
hardening suggestion, not a vulnerability. We are glad to receive those, but
|
||||
they are handled on the normal issue tracker, not as security advisories.
|
||||
|
||||
### Automated and AI-assisted reports
|
||||
|
||||
Output from static analysis, dependency scanners, fuzzers, or LLMs is welcome
|
||||
only when you have manually validated it and can supply a working reproduction
|
||||
against a supported version, per the requirements above. Raw tool output,
|
||||
speculative findings, or generated reports without a demonstrated exploit will
|
||||
be closed as hardening suggestions.
|
||||
|
||||
## Trust model
|
||||
|
||||
SeaweedFS is built to run with its cluster components (master, volume servers,
|
||||
and the raw filer API) on a trusted network. Those internal APIs are not an
|
||||
authentication boundary unless you explicitly enable a control (for example
|
||||
volume JWT or filer authentication) and that control is bypassed. Exposing an
|
||||
internal port directly to untrusted clients is a deployment mistake, not a
|
||||
vulnerability in SeaweedFS.
|
||||
|
||||
Reports are in scope when they cross a boundary SeaweedFS is meant to enforce,
|
||||
for example:
|
||||
|
||||
- Unauthenticated access to data or operations that require authentication
|
||||
- One S3 identity reading, writing, or deleting another identity's data
|
||||
- Privilege escalation from a normal S3 user to administrative capability
|
||||
- Bypass of Object Lock / retention where it is configured
|
||||
- Remotely triggered data corruption or loss
|
||||
|
||||
Reports are generally out of scope when they require:
|
||||
|
||||
- Direct access to an internal cluster port that is meant to be private
|
||||
- Full master, filer, or volume server access (already a full compromise)
|
||||
- An insecure example configuration rather than a documented secure setup
|
||||
- Local-only impact on a host the attacker already controls
|
||||
|
||||
## CVE assignment
|
||||
|
||||
When a report is confirmed, we publish an advisory and request the CVE through
|
||||
GitHub. CVEs assigned by third parties without coordinating with us, or for
|
||||
issues that do not cross a boundary described above, may be disputed.
|
||||
|
||||
## Response and disclosure
|
||||
|
||||
- We aim to acknowledge a valid report within a few business days.
|
||||
- We will investigate, work on a fix, and coordinate a disclosure timeline
|
||||
with you.
|
||||
- Please allow time for a fix before any public disclosure.
|
||||
- Please allow time for a fix before public disclosure
|
||||
- If you’re unsure whether something is a security issue, feel free to reach out
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary,
|
||||
# so arm64/arm/386 targets skip QEMU emulation of the whole compile.
|
||||
FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
@@ -14,12 +12,9 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
|
||||
git checkout $BRANCH) || \
|
||||
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
|
||||
echo "Available branches:" && git branch -a && exit 1))
|
||||
ARG TARGETOS TARGETARCH TARGETVARIANT
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& export GOOS=$TARGETOS GOARCH=$TARGETARCH \
|
||||
&& case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \
|
||||
&& CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed .
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
# Rust volume server: use pre-built binary from CI when available (placed in
|
||||
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
|
||||
|
||||
@@ -127,9 +127,6 @@ test_tarantool: tags = tarantool
|
||||
test_tarantool: build_tarantool_dev_env build
|
||||
docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
|
||||
|
||||
test_keycloak_s3: build
|
||||
docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
|
||||
|
||||
clean:
|
||||
rm ./weed
|
||||
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
services:
|
||||
keycloak:
|
||||
image: quay.io/keycloak/keycloak:26.0.7
|
||||
command: ["start-dev", "--import-realm"]
|
||||
environment:
|
||||
KC_BOOTSTRAP_ADMIN_USERNAME: admin
|
||||
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
|
||||
KC_HTTP_ENABLED: "true"
|
||||
KC_HOSTNAME: "http://keycloak:8080"
|
||||
KC_HOSTNAME_STRICT: "false"
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command:
|
||||
- -v=9
|
||||
- server
|
||||
- -ip=s3
|
||||
- -filer
|
||||
- -master.volumeSizeLimitMB=16
|
||||
- -volume
|
||||
- -volume.max=0
|
||||
- -volume.preStopSeconds=1
|
||||
- -s3
|
||||
- -s3.port=8333
|
||||
- -s3.config=/etc/seaweedfs/s3_config.json
|
||||
- -s3.iam.config=/etc/seaweedfs/iam_config.json
|
||||
environment:
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
|
||||
volumes:
|
||||
- ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
|
||||
- ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
|
||||
ports:
|
||||
- "8333:8333"
|
||||
depends_on:
|
||||
keycloak:
|
||||
condition: service_healthy
|
||||
@@ -1,11 +1,11 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.25.5
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.2
|
||||
cloud.google.com/go/storage v1.62.3
|
||||
cloud.google.com/go/storage v1.62.1
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -38,7 +38,7 @@ require (
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
@@ -46,7 +46,7 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/klauspost/reedsolomon v1.14.1
|
||||
github.com/klauspost/reedsolomon v1.14.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.8
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
@@ -85,51 +85,51 @@ require (
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.12
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.46.0
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.46.0
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.52.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/image v0.41.0
|
||||
golang.org/x/net v0.55.0
|
||||
golang.org/x/image v0.39.0
|
||||
golang.org/x/net v0.54.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.46.0
|
||||
golang.org/x/sys v0.45.0
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/tools v0.45.0 // indirect
|
||||
golang.org/x/tools v0.44.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.278.0
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/grpc v1.81.1
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.53.0
|
||||
modernc.org/sqlite v1.49.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.31.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.1020
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.2
|
||||
github.com/apache/iceberg-go v0.6.0
|
||||
github.com/a-h/templ v0.3.1001
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
|
||||
github.com/apache/iceberg-go v0.5.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.42.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.25
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.24
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/cognusion/imaging v1.0.3
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.9.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
@@ -143,21 +143,23 @@ require (
|
||||
github.com/parquet-go/parquet-go v0.30.1
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.11.0
|
||||
github.com/rclone/rclone v1.74.3
|
||||
github.com/rclone/rclone v1.74.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.21.0
|
||||
github.com/redis/go-redis/v9 v9.19.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.5
|
||||
github.com/shirou/gopsutil/v4 v4.26.3
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.43.0
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.4
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.141.0
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.12
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.11
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
)
|
||||
@@ -178,9 +180,9 @@ require (
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.6.0 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.23.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -198,28 +200,28 @@ require (
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.4 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/dave/dst v0.27.2 // indirect
|
||||
github.com/diskfs/go-diskfs v1.7.0 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/docker v28.5.2+incompatible // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.9.0 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/gookit/color v1.6.0 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/gopherjs/gopherjs v1.17.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/hamba/avro/v2 v2.31.0 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
@@ -228,21 +230,23 @@ require (
|
||||
github.com/jaegertracing/jaeger v1.47.0 // indirect
|
||||
github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect
|
||||
github.com/jtolds/gls v4.20.0+incompatible // indirect
|
||||
github.com/klauspost/asmfmt v1.3.2 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lib/pq v1.12.0 // indirect
|
||||
github.com/lib/pq v1.11.1 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/api v1.54.2 // indirect
|
||||
github.com/moby/moby/client v0.4.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/go-archive v0.1.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.0 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
@@ -251,19 +255,19 @@ require (
|
||||
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.83 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
github.com/stretchr/objx v0.5.3 // indirect
|
||||
github.com/substrait-io/substrait v0.87.0 // indirect
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.0 // indirect
|
||||
github.com/substrait-io/substrait-protobuf/go v0.85.0 // indirect
|
||||
github.com/twmb/avro v1.7.2 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/substrait-io/substrait v0.81.0 // indirect
|
||||
github.com/substrait-io/substrait-go/v7 v7.4.0 // indirect
|
||||
github.com/substrait-io/substrait-protobuf/go v0.81.0 // indirect
|
||||
github.com/twpayne/go-geom v1.6.1 // indirect
|
||||
github.com/twpayne/go-kml/v3 v3.2.1 // indirect
|
||||
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
|
||||
@@ -273,14 +277,15 @@ require (
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.36.0 // indirect
|
||||
golang.org/x/mod v0.35.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
@@ -292,8 +297,8 @@ require (
|
||||
cloud.google.com/go/iam v1.7.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
@@ -314,26 +319,27 @@ require (
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.43.3
|
||||
github.com/aws/smithy-go v1.27.2
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -344,7 +350,7 @@ require (
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
@@ -356,8 +362,8 @@ require (
|
||||
github.com/elastic/gosigar v0.14.3 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
@@ -376,7 +382,7 @@ require (
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/goccy/go-json v0.10.6 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
@@ -386,7 +392,7 @@ require (
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
@@ -420,8 +426,8 @@ require (
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.49.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
||||
github.com/nats-io/nats.go v1.48.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.12 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
@@ -435,7 +441,7 @@ require (
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.27
|
||||
github.com/pierrec/lz4/v4 v4.1.26
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
@@ -472,7 +478,7 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260428144813-1c07baab7f7b // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@@ -480,9 +486,9 @@ require (
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.12 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
@@ -492,7 +498,7 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.44.0
|
||||
golang.org/x/term v0.43.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
@@ -500,7 +506,7 @@ require (
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
@@ -517,9 +523,3 @@ require (
|
||||
// lib/go/thrift/framed_transport.go, which overflows int on 32-bit GOARCHes
|
||||
// (e.g. openbsd/arm, linux/arm). Pin to v0.22.0 until upstream fixes it.
|
||||
replace github.com/apache/thrift => github.com/apache/thrift v0.22.0
|
||||
|
||||
// tyler-smith/go-bip39 was deleted from GitHub, so `go mod download` fails for
|
||||
// anyone resolving it directly (GOPROXY=direct). It only reaches us transitively
|
||||
// through rclone's internxt backend, which calls IsMnemonicValid and NewSeed.
|
||||
// cosmos/go-bip39 is a maintained, API-compatible fork.
|
||||
replace github.com/tyler-smith/go-bip39 => github.com/cosmos/go-bip39 v1.0.0
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
# Usage:
|
||||
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
|
||||
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
|
||||
# curl -fsSL ... | bash -s -- --version v3.93 --dir /usr/local/bin
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, all (default: weed)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.37"
|
||||
appVersion: "4.29"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.37.0
|
||||
version: 4.29.0
|
||||
|
||||
@@ -384,4 +384,4 @@ helm install seaweedfs seaweedfs/seaweedfs \
|
||||
## Enterprise
|
||||
|
||||
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
|
||||
which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.
|
||||
which has a self-healing storage format with better data protection.
|
||||
|
||||
@@ -3666,291 +3666,6 @@
|
||||
],
|
||||
"title": "S3 Bucket Object Count",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 130
|
||||
},
|
||||
"id": 92,
|
||||
"panels": [],
|
||||
"title": "Filer Object Size Distribution",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Rate of new objects created, split by size range.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "objects/s",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "normal"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 131
|
||||
},
|
||||
"id": 93,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "8.1.2",
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "< 1KB",
|
||||
"refId": "A",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1KB - 100KB",
|
||||
"refId": "B",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100KB - 1MB",
|
||||
"refId": "C",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1MB - 100MB",
|
||||
"refId": "D",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100MB - 1GB",
|
||||
"refId": "E",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "> 1GB",
|
||||
"refId": "F",
|
||||
"step": 60
|
||||
}
|
||||
],
|
||||
"title": "Filer Object Write Rate by Size Range",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Objects created per size range over the selected time window.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 131
|
||||
},
|
||||
"id": 94,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"valueMode": "color"
|
||||
},
|
||||
"pluginVersion": "8.1.2",
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range]))",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "< 1KB",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1KB - 100KB",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100KB - 1MB",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1MB - 100MB",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100MB - 1GB",
|
||||
"refId": "E"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "> 1GB",
|
||||
"refId": "F"
|
||||
}
|
||||
],
|
||||
"title": "Filer Object Size Distribution",
|
||||
"type": "bargauge"
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
|
||||
@@ -81,9 +81,8 @@ spec:
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
{{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global.seaweedfs "component" .Values.allInOne "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- $clusterDefault := default "sw" (index $mergedExtraEnvironmentVars "WEED_CLUSTER_DEFAULT") }}
|
||||
{{- $envMerged := merge (.Values.global.seaweedfs.extraEnvironmentVars | default dict) (.Values.allInOne.extraEnvironmentVars | default dict) }}
|
||||
{{- $clusterDefault := default "sw" (index $envMerged "WEED_CLUSTER_DEFAULT") }}
|
||||
{{- $clusterUpper := upper $clusterDefault }}
|
||||
{{- $clusterMasterKey := printf "WEED_CLUSTER_%s_MASTER" $clusterUpper }}
|
||||
{{- $clusterFilerKey := printf "WEED_CLUSTER_%s_FILER" $clusterUpper }}
|
||||
@@ -101,8 +100,8 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.allInOne.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.allInOne.extraEnvironmentVars }}
|
||||
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
@@ -113,6 +112,20 @@ spec:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.seaweedfs.extraEnvironmentVars }}
|
||||
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
# Inject computed cluster endpoints for the default cluster
|
||||
- name: {{ $clusterMasterKey }}
|
||||
value: {{ include "seaweedfs.cluster.masterAddress" . | quote }}
|
||||
@@ -259,9 +272,6 @@ spec:
|
||||
{{- $authMethods := .Values.allInOne.sftp.authMethods | default .Values.sftp.authMethods }}
|
||||
{{- if $authMethods }}
|
||||
-sftp.authMethods={{ $authMethods }} \
|
||||
{{- if contains "certificate" $authMethods }}
|
||||
-sftp.trustedUserCAKeysFile=/etc/sw/sftp_ca/ca_user.pub \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $maxAuthTries := .Values.allInOne.sftp.maxAuthTries | default .Values.sftp.maxAuthTries }}
|
||||
{{- if $maxAuthTries }}
|
||||
@@ -309,12 +319,6 @@ spec:
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- $aioAuthMethods := .Values.allInOne.sftp.authMethods | default .Values.sftp.authMethods }}
|
||||
{{- if and $aioAuthMethods (contains "certificate" $aioAuthMethods) }}
|
||||
- mountPath: /etc/sw/sftp_ca
|
||||
name: config-sftp-ca
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.notificationConfig }}
|
||||
- name: notification-config
|
||||
@@ -450,18 +454,6 @@ spec:
|
||||
defaultMode: 420
|
||||
secretName: {{ default (printf "%s-sftp-secret" (include "seaweedfs.fullname" .)) (or .Values.allInOne.sftp.existingConfigSecret .Values.sftp.existingConfigSecret) }}
|
||||
{{- end }}
|
||||
{{- $aioAuthMethodsVol := .Values.allInOne.sftp.authMethods | default .Values.sftp.authMethods }}
|
||||
{{- if and $aioAuthMethodsVol (contains "certificate" $aioAuthMethodsVol) }}
|
||||
{{- $aioCASecret := or .Values.allInOne.sftp.existingCAKeysSecret .Values.sftp.existingCAKeysSecret }}
|
||||
{{- $aioCAKeys := or .Values.allInOne.sftp.trustedUserCAKeys .Values.sftp.trustedUserCAKeys }}
|
||||
{{- if and (not $aioCASecret) (not $aioCAKeys) }}
|
||||
{{- fail "allInOne.sftp.authMethods includes \"certificate\" but neither trustedUserCAKeys nor existingCAKeysSecret is set" }}
|
||||
{{- end }}
|
||||
- name: config-sftp-ca
|
||||
secret:
|
||||
defaultMode: 420
|
||||
secretName: {{ default (printf "%s-sftp-ca-secret" (include "seaweedfs.fullname" .)) $aioCASecret }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.notificationConfig }}
|
||||
- name: notification-config
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- /*
|
||||
Render a chart-managed Secret carrying the SSH user CA public key(s) for
|
||||
certificate-based SFTP authentication, but only when:
|
||||
- SFTP (standalone or all-in-one) is enabled,
|
||||
- "certificate" is in the relevant authMethods list,
|
||||
- the user provided inline CA keys via sftp.trustedUserCAKeys, and
|
||||
- no existingCAKeysSecret is set (which would supersede this Secret).
|
||||
*/}}
|
||||
{{- $sftpEnabled := or .Values.sftp.enabled (and .Values.allInOne.enabled .Values.allInOne.sftp.enabled) -}}
|
||||
{{- $authMethods := .Values.sftp.authMethods -}}
|
||||
{{- if and .Values.allInOne.enabled .Values.allInOne.sftp.authMethods -}}
|
||||
{{- $authMethods = .Values.allInOne.sftp.authMethods -}}
|
||||
{{- end -}}
|
||||
{{- $certInMethods := and $authMethods (contains "certificate" $authMethods) -}}
|
||||
{{- $inlineCAKeys := .Values.sftp.trustedUserCAKeys -}}
|
||||
{{- if and .Values.allInOne.enabled .Values.allInOne.sftp.trustedUserCAKeys -}}
|
||||
{{- $inlineCAKeys = .Values.allInOne.sftp.trustedUserCAKeys -}}
|
||||
{{- end -}}
|
||||
{{- $existingSecret := .Values.sftp.existingCAKeysSecret -}}
|
||||
{{- if and .Values.allInOne.enabled .Values.allInOne.sftp.existingCAKeysSecret -}}
|
||||
{{- $existingSecret = .Values.allInOne.sftp.existingCAKeysSecret -}}
|
||||
{{- end -}}
|
||||
{{- if and $sftpEnabled $certInMethods $inlineCAKeys (not $existingSecret) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
type: Opaque
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" . }}-sftp-ca-secret
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: sftp
|
||||
stringData:
|
||||
ca_user.pub: |
|
||||
{{ $inlineCAKeys | indent 4 }}
|
||||
{{- end }}
|
||||
@@ -135,9 +135,6 @@ spec:
|
||||
{{- end }}
|
||||
{{- if .Values.sftp.authMethods }}
|
||||
-authMethods={{ .Values.sftp.authMethods }} \
|
||||
{{- if contains "certificate" .Values.sftp.authMethods }}
|
||||
-trustedUserCAKeysFile=/etc/sw/sftp_ca/ca_user.pub \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.sftp.maxAuthTries }}
|
||||
-maxAuthTries={{ .Values.sftp.maxAuthTries }} \
|
||||
@@ -179,11 +176,6 @@ spec:
|
||||
- mountPath: /etc/sw/ssh
|
||||
name: config-ssh
|
||||
readOnly: true
|
||||
{{- if and .Values.sftp.authMethods (contains "certificate" .Values.sftp.authMethods) }}
|
||||
- mountPath: /etc/sw/sftp_ca
|
||||
name: config-sftp-ca
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
@@ -264,19 +256,6 @@ spec:
|
||||
{{- else }}
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-sftp-ssh-secret
|
||||
{{- end }}
|
||||
{{- if and .Values.sftp.authMethods (contains "certificate" .Values.sftp.authMethods) }}
|
||||
{{- if and (not .Values.sftp.existingCAKeysSecret) (not .Values.sftp.trustedUserCAKeys) }}
|
||||
{{- fail "sftp.authMethods includes \"certificate\" but neither sftp.trustedUserCAKeys nor sftp.existingCAKeysSecret is set" }}
|
||||
{{- end }}
|
||||
- name: config-sftp-ca
|
||||
secret:
|
||||
defaultMode: 420
|
||||
{{- if .Values.sftp.existingCAKeysSecret }}
|
||||
secretName: {{ .Values.sftp.existingCAKeysSecret }}
|
||||
{{- else }}
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-sftp-ca-secret
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.sftp.logs.type "hostPath" }}
|
||||
- name: logs
|
||||
hostPath:
|
||||
|
||||
@@ -333,13 +333,11 @@ Create the name of the service account to use
|
||||
{{- end -}}
|
||||
|
||||
{{/* True when security.toml should be rendered and mounted. volumeWrite is
|
||||
excluded unless its non-default expiration is configured. */}}
|
||||
excluded since it defaults to true. */}}
|
||||
{{- define "seaweedfs.securityConfigEnabled" -}}
|
||||
{{- $sec := (.Values.global.seaweedfs).securityConfig | default dict -}}
|
||||
{{- $jwt := $sec.jwtSigning | default dict -}}
|
||||
{{- $expiresAfterSeconds := $jwt.expiresAfterSeconds | default dict -}}
|
||||
{{- $volumeWriteExpirationConfigured := and $jwt.volumeWrite (gt (int $expiresAfterSeconds.volumeWrite) 0) -}}
|
||||
{{- if or .Values.global.seaweedfs.enableSecurity $volumeWriteExpirationConfigured $jwt.volumeRead $jwt.filerWrite $jwt.filerRead -}}
|
||||
{{- if or .Values.global.seaweedfs.enableSecurity $jwt.volumeRead $jwt.filerWrite $jwt.filerRead -}}
|
||||
true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -143,8 +143,6 @@ spec:
|
||||
{{- if kindIs "bool" .versioning }}
|
||||
{{- if .versioning }}
|
||||
{{- $bucketVersioning = "Enabled" }}
|
||||
{{- else }}
|
||||
{{- $bucketVersioning = "Suspended" }}
|
||||
{{- end }}
|
||||
{{- else if kindIs "string" .versioning }}
|
||||
{{- $versioningLower := lower .versioning }}
|
||||
|
||||
@@ -19,56 +19,40 @@ data:
|
||||
{{- $existing = lookup "v1" "ConfigMap" .Release.Namespace $legacyName }}
|
||||
{{- end }}
|
||||
{{- $securityConfig := fromToml (dig "data" "security.toml" "" $existing) }}
|
||||
{{- $securityConfigValues := .Values.global.seaweedfs.securityConfig | default dict }}
|
||||
{{- $jwtSigning := $securityConfigValues.jwtSigning | default dict }}
|
||||
{{- $expiresAfterSeconds := $jwtSigning.expiresAfterSeconds | default dict }}
|
||||
security.toml: |-
|
||||
# this file is read by master, volume server, and filer
|
||||
|
||||
{{- if $jwtSigning.volumeWrite }}
|
||||
{{- if .Values.global.seaweedfs.securityConfig.jwtSigning.volumeWrite }}
|
||||
# the jwt signing key is read by master and volume server
|
||||
# the jwt defaults to expire after 10 seconds
|
||||
# a jwt expires in 10 seconds
|
||||
[jwt.signing]
|
||||
key = "{{ dig "jwt" "signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.volumeWrite) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.volumeWrite }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if $jwtSigning.volumeRead }}
|
||||
{{- if .Values.global.seaweedfs.securityConfig.jwtSigning.volumeRead }}
|
||||
# this jwt signing key is read by master and volume server, and it is used for read operations:
|
||||
# - the Master server generates the JWT, which can be used to read a certain file on a volume server
|
||||
# - the Volume server validates the JWT on reading
|
||||
# the jwt defaults to expire after 60 seconds
|
||||
[jwt.signing.read]
|
||||
key = "{{ dig "jwt" "signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.volumeRead) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.volumeRead }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if $jwtSigning.filerWrite }}
|
||||
{{- if .Values.global.seaweedfs.securityConfig.jwtSigning.filerWrite }}
|
||||
# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# the jwt defaults to expire after 10 seconds
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing]
|
||||
key = "{{ dig "jwt" "filer_signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.filerWrite) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.filerWrite }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if $jwtSigning.filerRead }}
|
||||
{{- if .Values.global.seaweedfs.securityConfig.jwtSigning.filerRead }}
|
||||
# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on reading
|
||||
# the jwt defaults to expire after 60 seconds
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing.read]
|
||||
key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.filerRead) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.filerRead }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
|
||||
@@ -4,10 +4,6 @@
|
||||
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
|
||||
{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }}
|
||||
|
||||
{{- if and $volume.idx (eq ($volume.idx.type | toString) "emptyDir") }}
|
||||
{{- fail (printf "%s: idx.type \"emptyDir\" is not supported. An ephemeral index is wiped on every pod restart while the data persists, forcing a full index rebuild from the .dat on each start (a fatal crash on older versions). Use the default (idx: {}, kept next to the data) or a persistentVolumeClaim/hostPath/existingClaim for a separate persistent index." $volumeName) }}
|
||||
{{- end }}
|
||||
|
||||
{{- if $volume.enabled }}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
@@ -83,23 +79,7 @@ spec:
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
command: [ '/bin/sh', '-c' ]
|
||||
args:
|
||||
- |
|
||||
for dir in {{ range $index, $dir := $volume.dataDirs }}{{ if ne $index 0 }} {{ end }}/{{ $dir.name }}{{ end }}; do
|
||||
# Rebuild a .idx missing from both dirs (e.g. an index volume that lost files) from the .dat, else the server fatally exits on its idx check.
|
||||
for dat in "$dir"/*.dat; do
|
||||
[ -e "$dat" ] || continue
|
||||
base=${dat##*/}; base=${base%.dat}
|
||||
if [ ! -e "$dir/$base.idx" ] && [ ! -e "/idx/$base.idx" ]; then
|
||||
echo "rebuilding missing idx in $dir (trigger: volume $base)"
|
||||
weed fix "$dir"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ls "$dir"/*.idx >/dev/null 2>&1; then
|
||||
mv "$dir"/*.idx /idx/
|
||||
fi
|
||||
done
|
||||
args: [ '{{range $dir := $volume.dataDirs }}if ls /{{$dir.name}}/*.idx >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
|
||||
volumeMounts:
|
||||
- name: idx
|
||||
mountPath: /idx
|
||||
|
||||
@@ -26,13 +26,6 @@ global:
|
||||
volumeRead: false
|
||||
filerWrite: false
|
||||
filerRead: false
|
||||
# Positive values override SeaweedFS token lifetime defaults.
|
||||
# Zero keeps the runtime defaults: 10s for writes and 60s for reads.
|
||||
expiresAfterSeconds:
|
||||
volumeWrite: 0
|
||||
volumeRead: 0
|
||||
filerWrite: 0
|
||||
filerRead: 0
|
||||
# we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
|
||||
serviceAccountName: "seaweedfs"
|
||||
serviceAccountAnnotations: {}
|
||||
@@ -385,7 +378,7 @@ volume:
|
||||
enabled: true
|
||||
image: alpine/k8s:1.28.4
|
||||
|
||||
# idx (the volume index) may use a separate PERSISTENT volume:
|
||||
# idx can be defined by:
|
||||
#
|
||||
# idx:
|
||||
# type: "hostPath"
|
||||
@@ -404,14 +397,13 @@ volume:
|
||||
# type: "existingClaim"
|
||||
# claimName: "myClaim"
|
||||
#
|
||||
# "emptyDir" is rejected for idx (the chart fails to render): an ephemeral
|
||||
# index is wiped on every restart while the data persists, forcing a full
|
||||
# rebuild each start (a fatal crash on older versions). "logs" may still use
|
||||
# any of the above or emptyDir.
|
||||
# or
|
||||
#
|
||||
# Default {} keeps the index next to the data, so it persists with the data and
|
||||
# needs no separate volume. Recommended unless you must split the index onto
|
||||
# its own persistent storage.
|
||||
# idx:
|
||||
# type: "emptyDir"
|
||||
|
||||
# same applies to "logs"
|
||||
|
||||
idx: {}
|
||||
|
||||
# Resource requests, limits, etc. for the vol-move-idx initContainer. This
|
||||
@@ -927,7 +919,7 @@ filer:
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
|
||||
# objectLock enables S3 Object Lock (irreversible, forces versioning)
|
||||
# versioning: Enabled or Suspended (or bool true/false)
|
||||
# versioning: Enabled or Suspended (or true to enable)
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
# anonymousRead: true
|
||||
@@ -980,7 +972,7 @@ s3:
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
|
||||
# objectLock enables S3 Object Lock (irreversible, forces versioning)
|
||||
# versioning: Enabled or Suspended (or bool true/false)
|
||||
# versioning: Enabled or Suspended (or true to enable)
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
# anonymousRead: true
|
||||
@@ -1146,7 +1138,7 @@ sftp:
|
||||
# SSH server configuration
|
||||
sshPrivateKey: "/etc/sw/seaweedfs_sftp_ssh_private_key" # Path to the SSH private key file for host authentication
|
||||
hostKeysFolder: "/etc/sw/ssh" # path to folder containing SSH private key files for host authentication
|
||||
authMethods: "password,publickey" # Comma-separated list of allowed auth methods: password, publickey, certificate
|
||||
authMethods: "password,publickey" # Comma-separated list of allowed auth methods: password, publickey, keyboard-interactive
|
||||
maxAuthTries: 6 # Maximum number of authentication attempts per connection
|
||||
bannerMessage: "SeaweedFS SFTP Server" # Message displayed before authentication
|
||||
loginGraceTime: "2m" # Timeout for authentication
|
||||
@@ -1163,18 +1155,6 @@ sftp:
|
||||
# Set to the name of an existing kubernetes Secret with the list of ssh private keys for sftp
|
||||
existingSshConfigSecret: null
|
||||
|
||||
# SSH user-certificate authentication (CA-signed user certs). Mirrors
|
||||
# OpenSSH `TrustedUserCAKeys` and MinIO `--sftp=trusted-user-ca-key`.
|
||||
# Add "certificate" to `authMethods` to activate; when active, plain
|
||||
# public keys are rejected on the public-key channel.
|
||||
#
|
||||
# Inline CA public keys in OpenSSH authorized_keys format (one per
|
||||
# line). Ignored when `existingCAKeysSecret` is set.
|
||||
trustedUserCAKeys: ""
|
||||
# Set to the name of an existing kubernetes Secret carrying the CA
|
||||
# public keys under data key `ca_user.pub`.
|
||||
existingCAKeysSecret: null
|
||||
|
||||
# Additional resources
|
||||
sidecars: []
|
||||
initContainers: ""
|
||||
@@ -1534,7 +1514,7 @@ allInOne:
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
|
||||
# objectLock enables S3 Object Lock (irreversible, forces versioning)
|
||||
# versioning: Enabled or Suspended (or bool true/false)
|
||||
# versioning: Enabled or Suspended (or true to enable)
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
# anonymousRead: true
|
||||
@@ -1565,10 +1545,6 @@ allInOne:
|
||||
existingConfigSecret: null
|
||||
# Set to the name of an existing kubernetes Secret with the SSH keys
|
||||
existingSshConfigSecret: null
|
||||
# SSH user-certificate authentication. See sftp.trustedUserCAKeys above.
|
||||
# (null on either field inherits from the top-level sftp.* setting.)
|
||||
trustedUserCAKeys: null
|
||||
existingCAKeysSecret: null
|
||||
|
||||
# Service settings
|
||||
service:
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 746 B |
@@ -78,11 +78,6 @@ service SeaweedFiler {
|
||||
rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
|
||||
}
|
||||
|
||||
// List the metadata subscribers currently connected to this filer
|
||||
// (FUSE mounts, S3, filer.sync, peer filers, ...).
|
||||
rpc ListMetadataSubscribers (ListMetadataSubscribersRequest) returns (ListMetadataSubscribersResponse) {
|
||||
}
|
||||
|
||||
rpc KvGet (KvGetRequest) returns (KvGetResponse) {
|
||||
}
|
||||
|
||||
@@ -393,13 +388,7 @@ message PosixLockRequest {
|
||||
bool is_moved = 2;
|
||||
PosixLockOp op = 3;
|
||||
PosixLockRange lock = 4;
|
||||
// locks carries the full set a mount holds on key for a KEEP_ALIVE
|
||||
// re-assertion, so the current owner filer can rebuild its in-memory state
|
||||
// after an ownership change or restart. lock.sid identifies the session.
|
||||
repeated PosixLockRange locks = 5;
|
||||
// cooling_probe marks a dual-read a new owner sends to the previous owner
|
||||
// during a ring change, so the previous owner answers from local state
|
||||
// without itself cooling-off (no recursion).
|
||||
bool cooling_probe = 6;
|
||||
}
|
||||
|
||||
@@ -626,22 +615,6 @@ message SubscribeMetadataResponse {
|
||||
repeated SubscribeMetadataResponse events = 4; // batch of additional events (backlog catch-up)
|
||||
repeated LogFileChunkRef log_file_refs = 5; // log file chunk refs for client direct-read
|
||||
}
|
||||
message ListMetadataSubscribersRequest {
|
||||
repeated string client_types = 1; // optional filter by client type, e.g. "mount"; empty = all
|
||||
}
|
||||
message ListMetadataSubscribersResponse {
|
||||
repeated MetadataSubscriber subscribers = 1;
|
||||
}
|
||||
message MetadataSubscriber {
|
||||
string client_name = 1; // "<type>@<address>"
|
||||
string client_type = 2; // e.g. "mount", "sw-vfs", "s3", "filer:<addr>"
|
||||
string address = 3; // client peer address
|
||||
string path_prefix = 4; // subscribed path prefix
|
||||
int32 client_id = 5;
|
||||
int32 client_epoch = 6;
|
||||
int64 connected_at_ns = 7;
|
||||
string filer_address = 8; // the filer this subscriber is connected to
|
||||
}
|
||||
// A persisted log file that the client can read directly from volume servers.
|
||||
// The file format is: [4-byte size | protobuf LogEntry] repeated.
|
||||
// Each LogEntry.Data contains a marshaled SubscribeMetadataResponse.
|
||||
|
||||
+3115
-9744
File diff suppressed because it is too large
Load Diff
@@ -147,11 +147,7 @@ message VolumeEcShardInformationMessage {
|
||||
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
|
||||
uint64 file_count = 8; // total needles in the .ecx index (live + tombstoned)
|
||||
uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
|
||||
// encode-run identity (unix nanos) from the .vif EcShardConfig; lets the admin
|
||||
// group shards by encode generation. Numbered 14 (not 10) to skip the
|
||||
// enterprise fork's reserved 10-13.
|
||||
int64 encode_ts_ns = 14;
|
||||
// fields 15-19 reserved for future upstream open-source additions.
|
||||
// fields 10-19 reserved for future upstream open-source additions.
|
||||
// fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
|
||||
// and must not be used here without coordination.
|
||||
}
|
||||
|
||||
@@ -442,7 +442,6 @@ message VolumeEcShardsCopyRequest {
|
||||
bool copy_ecj_file = 6;
|
||||
bool copy_vif_file = 7;
|
||||
uint32 disk_id = 8; // Target disk ID for storing EC shards
|
||||
bool copy_ecsum_file = 9; // copy the bitrot checksum sidecar (.ecsum) when present; tolerant of a missing source (no-op), since this non-2PC path has no Prepare backstop
|
||||
}
|
||||
message VolumeEcShardsCopyResponse {
|
||||
}
|
||||
@@ -451,11 +450,8 @@ message VolumeEcShardsDeleteRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
bool full_teardown = 4; // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
|
||||
int64 encode_ts_ns = 5; // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
|
||||
}
|
||||
message VolumeEcShardsDeleteResponse {
|
||||
bool full_teardown_done = 1; // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op
|
||||
}
|
||||
|
||||
message VolumeEcShardsMountRequest {
|
||||
@@ -463,7 +459,6 @@ message VolumeEcShardsMountRequest {
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
|
||||
bool recover_missing_index = 5; // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104)
|
||||
}
|
||||
message VolumeEcShardsMountResponse {
|
||||
}
|
||||
@@ -471,7 +466,6 @@ message VolumeEcShardsMountResponse {
|
||||
message VolumeEcShardsUnmountRequest {
|
||||
uint32 volume_id = 1;
|
||||
repeated uint32 shard_ids = 3;
|
||||
int64 encode_ts_ns = 4; // generation fence: skip a disk whose mounted EC volume is this generation or newer (0 = unfenced, unmount all)
|
||||
}
|
||||
message VolumeEcShardsUnmountResponse {
|
||||
}
|
||||
@@ -482,13 +476,10 @@ message VolumeEcShardReadRequest {
|
||||
int64 offset = 3;
|
||||
int64 size = 4;
|
||||
uint64 file_key = 5;
|
||||
reserved 6;
|
||||
int64 encode_ts_ns = 7; // caller's expected encode time; the server rejects a shard from a different encode run
|
||||
}
|
||||
message VolumeEcShardReadResponse {
|
||||
bytes data = 1;
|
||||
bool is_deleted = 2;
|
||||
int64 encode_ts_ns = 3; // identity of the shard actually served; client rejects a mismatch (0 = pre-upgrade server)
|
||||
}
|
||||
|
||||
message VolumeEcBlobDeleteRequest {
|
||||
@@ -586,30 +577,6 @@ message VolumeInfo {
|
||||
message EcShardConfig {
|
||||
uint32 data_shards = 1; // Number of data shards (e.g., 10)
|
||||
uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
|
||||
int64 encode_ts_ns = 3; // encode time (unix nanos); a read served from a shard of a different encode run is rejected
|
||||
}
|
||||
// EcBitrotProtection is the entire content of a bitrot checksum sidecar
|
||||
// (<base>.ecsum for the legacy generation, <base>.ecsum.v<N> for vacuum
|
||||
// generation N): per-shard, per-block CRC32C so a CHECKSUM scrub can detect
|
||||
// silent bit rot in any shard (including parity) without decoding. Field
|
||||
// numbers and types match weed/pb/volume_server.proto byte-for-byte so the
|
||||
// serialized sidecar payload is wire-identical across the Go and Rust binaries.
|
||||
message EcBitrotProtection {
|
||||
ChecksumAlgorithm algorithm = 1; // CRC32C (Castagnoli)
|
||||
uint32 block_size = 2; // bytes per checksum block; default 16777216 (16 MiB), a power-of-two multiple of 1 MiB
|
||||
uint32 generation = 3; // EC vacuum generation these checksums describe (0 = legacy/fresh); must match the sidecar filename version
|
||||
EcShardConfig ec_shard_config = 4; // data/parity shard counts at encode time
|
||||
repeated EcShardChecksums shards = 5; // one entry per shard id in the active layout
|
||||
bytes encode_uuid = 6; // random per-encode identity, for stale-sidecar detection across in-place re-encodes
|
||||
}
|
||||
message EcShardChecksums {
|
||||
uint32 shard_id = 1; // 0..MaxShardCount-1 (custom EC ratios go up to 32)
|
||||
int64 covered_size = 2; // shard byte length these checksums cover (must equal the on-disk shard length)
|
||||
bytes block_crc32c = 3; // packed little-endian uint32[] = ceil(covered_size/block_size) entries
|
||||
}
|
||||
enum ChecksumAlgorithm {
|
||||
CHECKSUM_NONE = 0;
|
||||
CHECKSUM_CRC32C = 1;
|
||||
}
|
||||
message OldVersionVolumeInfo {
|
||||
repeated RemoteFile files = 1;
|
||||
@@ -688,7 +655,6 @@ enum VolumeScrubMode {
|
||||
INDEX = 1;
|
||||
FULL = 2;
|
||||
LOCAL = 3;
|
||||
CHECKSUM = 4; // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar
|
||||
}
|
||||
|
||||
message ScrubVolumeRequest {
|
||||
|
||||
@@ -188,12 +188,6 @@ pub struct Cli {
|
||||
#[arg(long = "securityFile", default_value = "")]
|
||||
pub security_file: String,
|
||||
|
||||
/// If true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints
|
||||
/// including loopback / link-local hosts. Default rejects internal /
|
||||
/// metadata endpoints.
|
||||
#[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)]
|
||||
pub allow_untrusted_remote_endpoints: bool,
|
||||
|
||||
/// A file of command line options, each line in optionName=optionValue format.
|
||||
#[arg(long = "options", default_value = "")]
|
||||
pub options: String,
|
||||
@@ -264,9 +258,6 @@ pub struct VolumeServerConfig {
|
||||
pub enable_write_queue: bool,
|
||||
/// Path to security.toml — stored for SIGHUP reload.
|
||||
pub security_file: String,
|
||||
/// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
|
||||
/// (allows loopback / link-local / metadata hosts).
|
||||
pub allow_untrusted_remote_endpoints: bool,
|
||||
}
|
||||
|
||||
pub use crate::storage::needle_map::NeedleMapKind;
|
||||
@@ -813,7 +804,6 @@ fn resolve_config(cli: Cli) -> VolumeServerConfig {
|
||||
.map(|v| v == "1" || v == "true")
|
||||
.unwrap_or(false),
|
||||
security_file: cli.security_file,
|
||||
allow_untrusted_remote_endpoints: cli.allow_untrusted_remote_endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -327,7 +327,6 @@ async fn run(
|
||||
seaweed_volume::remote_storage::s3_tier::S3TierRegistry::new(),
|
||||
),
|
||||
read_mode: config.read_mode,
|
||||
allow_untrusted_remote_endpoints: config.allow_untrusted_remote_endpoints,
|
||||
master_url,
|
||||
master_urls,
|
||||
seed_master_set,
|
||||
@@ -546,12 +545,6 @@ async fn run(
|
||||
whitelist.extend(sec.guard_white_list.iter().cloned());
|
||||
let mut guard = state_reload.guard.write().unwrap();
|
||||
guard.update_whitelist(&whitelist);
|
||||
guard.update_signing_keys(
|
||||
SigningKey(sec.jwt_signing_key),
|
||||
sec.jwt_signing_expires,
|
||||
SigningKey(sec.jwt_read_signing_key),
|
||||
sec.jwt_read_signing_expires,
|
||||
);
|
||||
}
|
||||
|
||||
// Trigger heartbeat to report new volumes
|
||||
|
||||
@@ -424,10 +424,6 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_push_metrics_once() {
|
||||
register_metrics();
|
||||
// A CounterVec with no children emits nothing, so create a labelset
|
||||
// instead of depending on another test having touched the counter
|
||||
// first (test order is nondeterministic).
|
||||
REQUEST_COUNTER.with_label_values(&["GET", "200"]).inc();
|
||||
|
||||
let captured = Arc::new(Mutex::new(None::<String>));
|
||||
let captured_clone = captured.clone();
|
||||
|
||||
@@ -1,359 +0,0 @@
|
||||
//! SSRF guard for caller-supplied remote storage endpoints.
|
||||
//!
|
||||
//! `FetchAndWriteNeedle` accepts an S3 endpoint URL chosen by the caller and
|
||||
//! dials it directly from the volume server, which typically has network
|
||||
//! access to cluster-internal hosts. Without validation a caller could point
|
||||
//! the server at loopback / link-local / RFC 1918 / cloud-metadata addresses
|
||||
//! and read internal services (SSRF). This mirrors the Go volume server's
|
||||
//! `validateRemoteEndpoint` (`weed/server/volume_grpc_remote.go`); operators
|
||||
//! that legitimately fetch from private hosts opt out with
|
||||
//! `-volume.allowUntrustedRemoteEndpoints`.
|
||||
//!
|
||||
//! NOTE: the Go server additionally pins the validated endpoint against DNS
|
||||
//! rebinding with a custom dialer that re-checks the resolved IP at TCP connect
|
||||
//! time (`guardedDialer`). The `aws-sdk-s3` client used here builds its own
|
||||
//! connector, so that connect-time re-validation is not yet ported; the
|
||||
//! up-front resolve-and-check below still blocks the common SSRF vectors. A
|
||||
//! narrow TOCTOU window (a hostname that resolves to a public IP here and then
|
||||
//! flips to a blocked one when the SDK dials) remains as a follow-up.
|
||||
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
/// AWS/Azure/GCP IPv4 instance-metadata-service (IMDS) address. It is
|
||||
/// link-local and thus already covered by [`is_link_local`], but is named
|
||||
/// explicitly so the rejection reason is unambiguous in logs.
|
||||
const IMDS_IPV4: IpAddr = IpAddr::V4(Ipv4Addr::new(169, 254, 169, 254));
|
||||
|
||||
/// Hostnames that target cloud instance metadata services, blocked regardless
|
||||
/// of how they resolve because some environments alias the IMDS address under
|
||||
/// a name.
|
||||
fn is_blocked_imds_host(host: &str) -> bool {
|
||||
matches!(host, "metadata.google.internal" | "metadata")
|
||||
}
|
||||
|
||||
/// Whether `ip` is in a link-local range: IPv4 169.254.0.0/16, IPv6 fe80::/10
|
||||
/// (unicast) and interface-local / link-local multicast (scope 1 and 2).
|
||||
fn is_link_local(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => v4.is_link_local(),
|
||||
IpAddr::V6(v6) => {
|
||||
let seg0 = v6.segments()[0];
|
||||
if (seg0 & 0xffc0) == 0xfe80 {
|
||||
return true; // fe80::/10 link-local unicast
|
||||
}
|
||||
if (seg0 & 0xff00) == 0xff00 {
|
||||
// ffXS:: multicast; reject interface-local (scope 1) and
|
||||
// link-local (scope 2) the way Go's IsInterfaceLocalMulticast /
|
||||
// IsLinkLocalMulticast do.
|
||||
let scope = seg0 & 0x000f;
|
||||
return scope == 1 || scope == 2;
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `ip` is in a private range: IPv4 RFC 1918, IPv6 fc00::/7 unique-local
|
||||
/// (matching Go's `net.IP.IsPrivate`).
|
||||
fn is_private(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => v4.is_private(),
|
||||
IpAddr::V6(v6) => (v6.segments()[0] & 0xfe00) == 0xfc00,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `ip` is in the RFC 6598 carrier-grade NAT range (100.64.0.0/10).
|
||||
/// The IPv4 private check does not cover CGNAT, so check it explicitly.
|
||||
fn is_cgnat(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
let o = v4.octets();
|
||||
o[0] == 100 && (o[1] & 0xc0) == 0x40 // second octet 64..=127
|
||||
}
|
||||
IpAddr::V6(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an error if `ip` is not safe to dial from a server that can reach
|
||||
/// cluster-internal hosts. Mirrors Go's `checkBlockedIP`.
|
||||
pub fn check_blocked_ip(endpoint: &str, ip: IpAddr) -> Result<(), String> {
|
||||
// Normalize IPv4-mapped IPv6 (`::ffff:a.b.c.d`) to its IPv4 form so the
|
||||
// IPv4 deny rules apply. The OS routes these to the embedded IPv4 address,
|
||||
// so without this `::ffff:127.0.0.1` / `::ffff:169.254.169.254` would slip
|
||||
// past the IPv6-only checks. Mirrors Go's reliance on net.IP.To4().
|
||||
let ip = match ip {
|
||||
IpAddr::V6(v6) => match v6.to_ipv4_mapped() {
|
||||
Some(v4) => IpAddr::V4(v4),
|
||||
None => IpAddr::V6(v6),
|
||||
},
|
||||
other => other,
|
||||
};
|
||||
if ip == IMDS_IPV4 {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} targets instance metadata service {}",
|
||||
endpoint, ip
|
||||
));
|
||||
}
|
||||
if ip.is_loopback() {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} resolves to loopback address {}",
|
||||
endpoint, ip
|
||||
));
|
||||
}
|
||||
if ip.is_unspecified() {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} resolves to unspecified address {}",
|
||||
endpoint, ip
|
||||
));
|
||||
}
|
||||
if is_link_local(ip) {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} resolves to link-local address {}",
|
||||
endpoint, ip
|
||||
));
|
||||
}
|
||||
if is_private(ip) {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} resolves to private address {}",
|
||||
endpoint, ip
|
||||
));
|
||||
}
|
||||
if is_cgnat(ip) {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} resolves to CGNAT address {}",
|
||||
endpoint, ip
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outcome of the synchronous, DNS-free portion of endpoint validation.
|
||||
#[derive(Debug)]
|
||||
enum HostCheck {
|
||||
/// Host was a literal IP and already passed [`check_blocked_ip`].
|
||||
Validated,
|
||||
/// Host is a name that must be resolved and each address checked.
|
||||
NeedsResolution(String),
|
||||
}
|
||||
|
||||
/// Validate the scheme and host without touching DNS. Pure and unit-testable.
|
||||
fn precheck_endpoint(endpoint: &str) -> Result<HostCheck, String> {
|
||||
let trimmed = endpoint.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err("remote endpoint is empty".to_string());
|
||||
}
|
||||
|
||||
let Some(scheme_end) = trimmed.find("://") else {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} must use http or https",
|
||||
endpoint
|
||||
));
|
||||
};
|
||||
let scheme = trimmed[..scheme_end].to_ascii_lowercase();
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} must use http or https, got {:?}",
|
||||
endpoint,
|
||||
&trimmed[..scheme_end]
|
||||
));
|
||||
}
|
||||
|
||||
// Authority is everything up to the first '/', '?', or '#'.
|
||||
let after = &trimmed[scheme_end + 3..];
|
||||
let authority_end = after
|
||||
.find(|c| c == '/' || c == '?' || c == '#')
|
||||
.unwrap_or(after.len());
|
||||
let authority = &after[..authority_end];
|
||||
|
||||
// Strip optional userinfo ("user:pass@").
|
||||
let host_port = match authority.rfind('@') {
|
||||
Some(at) => &authority[at + 1..],
|
||||
None => authority,
|
||||
};
|
||||
|
||||
// Extract the host, handling bracketed IPv6 literals and host:port.
|
||||
let host = if let Some(rest) = host_port.strip_prefix('[') {
|
||||
match rest.find(']') {
|
||||
Some(end) => &rest[..end],
|
||||
None => {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} has a malformed IPv6 host",
|
||||
endpoint
|
||||
))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match host_port.find(':') {
|
||||
Some(colon) => &host_port[..colon],
|
||||
None => host_port,
|
||||
}
|
||||
};
|
||||
|
||||
if host.is_empty() {
|
||||
return Err(format!("remote endpoint {:?} has no host", endpoint));
|
||||
}
|
||||
|
||||
if is_blocked_imds_host(&host.to_ascii_lowercase()) {
|
||||
return Err(format!(
|
||||
"remote endpoint {:?} targets instance metadata service",
|
||||
endpoint
|
||||
));
|
||||
}
|
||||
|
||||
if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
check_blocked_ip(endpoint, ip)?;
|
||||
return Ok(HostCheck::Validated);
|
||||
}
|
||||
|
||||
Ok(HostCheck::NeedsResolution(host.to_string()))
|
||||
}
|
||||
|
||||
/// Resolve `host` to its addresses with a short timeout, matching Go's 2s
|
||||
/// resolver deadline.
|
||||
async fn resolve_host(host: &str) -> Result<Vec<IpAddr>, String> {
|
||||
let lookup = tokio::net::lookup_host((host, 0u16));
|
||||
let addrs = tokio::time::timeout(std::time::Duration::from_secs(2), lookup)
|
||||
.await
|
||||
.map_err(|_| format!("resolve remote endpoint host {:?}: timed out", host))?
|
||||
.map_err(|e| format!("resolve remote endpoint host {:?}: {}", host, e))?;
|
||||
Ok(addrs.map(|sock| sock.ip()).collect())
|
||||
}
|
||||
|
||||
/// Returns an error if `endpoint` is not safe to dial from a server with access
|
||||
/// to cluster-internal hosts. Rejects empty / non-http(s) endpoints,
|
||||
/// loopback / unspecified / link-local / RFC 1918 / CGNAT addresses, and
|
||||
/// well-known IMDS hostnames. Hostnames are resolved and every returned address
|
||||
/// is checked. Mirrors Go's `validateRemoteEndpoint`.
|
||||
pub async fn validate_remote_endpoint(endpoint: &str) -> Result<(), String> {
|
||||
match precheck_endpoint(endpoint)? {
|
||||
HostCheck::Validated => Ok(()),
|
||||
HostCheck::NeedsResolution(host) => {
|
||||
let addrs = resolve_host(&host).await?;
|
||||
if addrs.is_empty() {
|
||||
return Err(format!(
|
||||
"resolve remote endpoint host {:?}: no addresses",
|
||||
host
|
||||
));
|
||||
}
|
||||
for ip in addrs {
|
||||
check_blocked_ip(endpoint, ip)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ip(s: &str) -> IpAddr {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_blocked_literal_endpoints() {
|
||||
let cases = [
|
||||
("http://127.0.0.1:8080", "loopback"),
|
||||
("http://[::1]:8080", "loopback"),
|
||||
("http://169.254.169.254/", "metadata"),
|
||||
("http://0.0.0.0/", "unspecified"),
|
||||
("http://[fe80::1]/", "link-local"),
|
||||
("http://10.0.0.1/", "private"),
|
||||
("http://172.16.5.5/", "private"),
|
||||
("http://192.168.0.1/", "private"),
|
||||
("http://100.64.0.1/", "CGNAT"),
|
||||
];
|
||||
for (endpoint, want) in cases {
|
||||
let err = precheck_endpoint(endpoint).expect_err(endpoint);
|
||||
assert!(err.contains(want), "{endpoint}: {err:?} missing {want:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_and_bad_scheme() {
|
||||
assert!(precheck_endpoint("").unwrap_err().contains("empty"));
|
||||
assert!(precheck_endpoint("ftp://example.com/")
|
||||
.unwrap_err()
|
||||
.contains("http or https"));
|
||||
assert!(precheck_endpoint("example.com/")
|
||||
.unwrap_err()
|
||||
.contains("http or https"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_imds_hostnames() {
|
||||
assert!(precheck_endpoint("http://metadata.google.internal/")
|
||||
.unwrap_err()
|
||||
.contains("metadata service"));
|
||||
assert!(precheck_endpoint("http://metadata/")
|
||||
.unwrap_err()
|
||||
.contains("metadata service"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_public_literal_and_defers_hostname() {
|
||||
assert!(matches!(
|
||||
precheck_endpoint("https://52.216.10.10/"),
|
||||
Ok(HostCheck::Validated)
|
||||
));
|
||||
match precheck_endpoint("https://s3.us-east-1.amazonaws.com/") {
|
||||
Ok(HostCheck::NeedsResolution(host)) => {
|
||||
assert_eq!(host, "s3.us-east-1.amazonaws.com")
|
||||
}
|
||||
other => panic!("expected resolution, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_blocked_ip_matches_resolved_categories() {
|
||||
// Mirror Go's "host resolves to X" cases at the address level.
|
||||
assert!(check_blocked_ip("e", ip("127.0.0.1"))
|
||||
.unwrap_err()
|
||||
.contains("loopback"));
|
||||
assert!(check_blocked_ip("e", ip("169.254.10.20"))
|
||||
.unwrap_err()
|
||||
.contains("link-local"));
|
||||
assert!(check_blocked_ip("e", ip("10.1.2.3"))
|
||||
.unwrap_err()
|
||||
.contains("private"));
|
||||
assert!(check_blocked_ip("e", ip("172.20.0.5"))
|
||||
.unwrap_err()
|
||||
.contains("private"));
|
||||
assert!(check_blocked_ip("e", ip("192.168.1.1"))
|
||||
.unwrap_err()
|
||||
.contains("private"));
|
||||
assert!(check_blocked_ip("e", ip("100.64.0.42"))
|
||||
.unwrap_err()
|
||||
.contains("CGNAT"));
|
||||
assert!(check_blocked_ip("e", ip("fc00::1"))
|
||||
.unwrap_err()
|
||||
.contains("private"));
|
||||
assert!(check_blocked_ip("e", ip("52.216.10.10")).is_ok());
|
||||
assert!(check_blocked_ip("e", ip("2606:4700:4700::1111")).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_ipv4_mapped_ipv6() {
|
||||
// IPv4-mapped IPv6 must be unmapped so the IPv4 rules catch it.
|
||||
assert!(check_blocked_ip("e", ip("::ffff:127.0.0.1"))
|
||||
.unwrap_err()
|
||||
.contains("loopback"));
|
||||
assert!(check_blocked_ip("e", ip("::ffff:169.254.169.254"))
|
||||
.unwrap_err()
|
||||
.contains("metadata"));
|
||||
assert!(check_blocked_ip("e", ip("::ffff:10.0.0.1"))
|
||||
.unwrap_err()
|
||||
.contains("private"));
|
||||
// A mapped public address still passes, and genuine IPv6 loopback is
|
||||
// still caught by the V6 path.
|
||||
assert!(check_blocked_ip("e", ip("::ffff:52.216.10.10")).is_ok());
|
||||
assert!(check_blocked_ip("e", ip("::1"))
|
||||
.unwrap_err()
|
||||
.contains("loopback"));
|
||||
// Bracketed mapped literal via the full endpoint path.
|
||||
assert!(precheck_endpoint("http://[::ffff:127.0.0.1]/")
|
||||
.unwrap_err()
|
||||
.contains("loopback"));
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,9 @@
|
||||
//! Provides a trait-based abstraction over cloud storage providers (S3, GCS, Azure, etc.)
|
||||
//! and a registry to create clients from protobuf RemoteConf messages.
|
||||
|
||||
pub mod endpoint_guard;
|
||||
pub mod s3;
|
||||
pub mod s3_tier;
|
||||
|
||||
pub use endpoint_guard::validate_remote_endpoint;
|
||||
|
||||
use crate::pb::remote_pb::{RemoteConf, RemoteStorageLocation};
|
||||
|
||||
/// Error type for remote storage operations.
|
||||
@@ -89,26 +86,6 @@ pub fn make_remote_storage_client(
|
||||
}
|
||||
}
|
||||
|
||||
/// Endpoint URL that the volume server would dial directly for `conf`, or
|
||||
/// `None` for non-S3-compatible backends. Every type handled here routes
|
||||
/// through [`s3::S3RemoteStorageClient`] with a caller-supplied endpoint, so
|
||||
/// the SSRF guard must validate each one. Keep this match in sync with
|
||||
/// [`make_remote_storage_client`].
|
||||
pub fn s3_compatible_endpoint(conf: &RemoteConf) -> Option<&str> {
|
||||
match conf.r#type.as_str() {
|
||||
"s3" => Some(&conf.s3_endpoint),
|
||||
"wasabi" => Some(&conf.wasabi_endpoint),
|
||||
"backblaze" => Some(&conf.backblaze_endpoint),
|
||||
"aliyun" => Some(&conf.aliyun_endpoint),
|
||||
"tencent" => Some(&conf.tencent_endpoint),
|
||||
"baidu" => Some(&conf.baidu_endpoint),
|
||||
"filebase" => Some(&conf.filebase_endpoint),
|
||||
"storj" => Some(&conf.storj_endpoint),
|
||||
"contabo" => Some(&conf.contabo_endpoint),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract S3-compatible credentials from a RemoteConf based on its type.
|
||||
fn extract_s3_credentials(conf: &RemoteConf) -> (String, String, String, String) {
|
||||
match conf.r#type.as_str() {
|
||||
@@ -178,37 +155,3 @@ fn extract_s3_credentials(conf: &RemoteConf) -> (String, String, String, String)
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn s3_compatible_endpoint_covers_all_s3_backends() {
|
||||
let s3 = RemoteConf {
|
||||
r#type: "s3".to_string(),
|
||||
s3_endpoint: "http://s3.internal".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(s3_compatible_endpoint(&s3), Some("http://s3.internal"));
|
||||
|
||||
// A non-"s3" S3-compatible type still surfaces its own endpoint, so the
|
||||
// SSRF guard cannot be bypassed by picking a different alias.
|
||||
let wasabi = RemoteConf {
|
||||
r#type: "wasabi".to_string(),
|
||||
wasabi_endpoint: "http://wasabi.internal".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
s3_compatible_endpoint(&wasabi),
|
||||
Some("http://wasabi.internal")
|
||||
);
|
||||
|
||||
// Non-S3 backends do not dial a caller-supplied URL directly.
|
||||
let gcs = RemoteConf {
|
||||
r#type: "gcs".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(s3_compatible_endpoint(&gcs), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,18 +364,6 @@ impl S3TierBackend {
|
||||
.map_err(|e| format!("download task panicked: {}", e))??;
|
||||
}
|
||||
|
||||
// fsync the file so its content is durable before the caller trims the .vif
|
||||
// and deletes the remote object (matches Go's DownloadFile f.Sync()).
|
||||
let synced = tokio::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(dest_path)
|
||||
.await
|
||||
.map_err(|e| format!("failed to open {} for fsync: {}", dest_path, e))?;
|
||||
synced
|
||||
.sync_all()
|
||||
.await
|
||||
.map_err(|e| format!("failed to fsync {}: {}", dest_path, e))?;
|
||||
|
||||
Ok(file_size)
|
||||
}
|
||||
|
||||
|
||||
@@ -172,26 +172,6 @@ impl Guard {
|
||||
self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
|
||||
}
|
||||
|
||||
/// Refresh the JWT signing keys and their expirations in place so operators
|
||||
/// can rotate keys via SIGHUP without restarting the process. Mirrors Go's
|
||||
/// `Guard::UpdateSigningKeys`: it swaps the fields and recomputes
|
||||
/// `is_write_active`.
|
||||
pub fn update_signing_keys(
|
||||
&mut self,
|
||||
signing_key: SigningKey,
|
||||
expires_after_sec: i64,
|
||||
read_signing_key: SigningKey,
|
||||
read_expires_after_sec: i64,
|
||||
) {
|
||||
self.signing_key = signing_key;
|
||||
self.expires_after_sec = expires_after_sec;
|
||||
self.read_signing_key = read_signing_key;
|
||||
self.read_expires_after_sec = read_expires_after_sec;
|
||||
|
||||
let is_empty_whitelist = self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty();
|
||||
self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
|
||||
}
|
||||
|
||||
/// Check if a remote IP is in the whitelist.
|
||||
/// Returns true if write security is inactive (no whitelist and no signing key),
|
||||
/// if the whitelist is empty, or if the IP matches.
|
||||
@@ -483,43 +463,6 @@ mod tests {
|
||||
assert!(matches!(err, Err(JwtError::FileIdMismatch { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_signing_keys_rotates() {
|
||||
let mut guard = Guard::new(
|
||||
&[],
|
||||
SigningKey::from_string("old-write"),
|
||||
10,
|
||||
SigningKey::from_string("old-read"),
|
||||
60,
|
||||
);
|
||||
|
||||
guard.update_signing_keys(
|
||||
SigningKey::from_string("new-write"),
|
||||
11,
|
||||
SigningKey::from_string("new-read"),
|
||||
61,
|
||||
);
|
||||
|
||||
let token = gen_jwt(&guard.signing_key, 60, "3,01637037d6").unwrap();
|
||||
// Validates with the rotated key, not the old one.
|
||||
assert!(decode_jwt(&guard.signing_key, &token).is_ok());
|
||||
assert!(decode_jwt(&SigningKey::from_string("old-write"), &token).is_err());
|
||||
assert_eq!(guard.expires_after_sec, 11);
|
||||
assert_eq!(guard.read_expires_after_sec, 61);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_signing_keys_toggles_write_active() {
|
||||
let mut guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
assert!(guard.check_jwt(None, true).is_ok());
|
||||
|
||||
guard.update_signing_keys(SigningKey::from_string("write"), 10, SigningKey(vec![]), 0);
|
||||
assert!(guard.is_write_active);
|
||||
|
||||
guard.update_signing_keys(SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
assert!(!guard.is_write_active);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_host() {
|
||||
assert_eq!(extract_host("192.168.1.1:8080"), "192.168.1.1");
|
||||
|
||||
@@ -189,7 +189,6 @@ mod tests {
|
||||
white_list: vec![],
|
||||
fix_jpg_orientation: false,
|
||||
read_mode: ReadMode::Local,
|
||||
allow_untrusted_remote_endpoints: false,
|
||||
cpu_profile: String::new(),
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -23,27 +23,6 @@ use crate::pb::volume_server_pb;
|
||||
use crate::storage::needle::needle::Needle;
|
||||
use crate::storage::types::*;
|
||||
|
||||
/// Slack added over the configured file-size limit when bounding the raw
|
||||
/// request body, to allow for multipart/form-data framing overhead. The exact
|
||||
/// per-file limit is still enforced on the parsed data after multipart parsing.
|
||||
const UPLOAD_BODY_OVERHEAD: usize = 16 * 1024 * 1024; // 16 MiB
|
||||
|
||||
/// Upper bound on bytes we will materialize in memory for a single request when
|
||||
/// expanding stored content (gzip decompression or chunk-manifest assembly).
|
||||
/// Guards against gzip bombs and crafted/oversized manifest sizes OOM-killing
|
||||
/// the server; legitimate large objects are read via client-side chunk fetches.
|
||||
const MAX_EXPANSION_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
|
||||
|
||||
/// Why `maybe_decompress_gzip` failed, so callers can distinguish a recoverable
|
||||
/// "not valid gzip, use the raw bytes" from "the bomb cap was hit, reject it".
|
||||
#[derive(Debug)]
|
||||
enum GunzipError {
|
||||
/// Input was not valid gzip / decode failed — callers may fall back to raw.
|
||||
Decode,
|
||||
/// Decompressed output would exceed `MAX_EXPANSION_BYTES` — must be rejected.
|
||||
TooLarge,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Inflight Throttle Guard
|
||||
// ============================================================================
|
||||
@@ -356,23 +335,12 @@ fn parse_url_path(path: &str) -> Option<(VolumeId, NeedleId, Cookie)> {
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
struct VolumeLocation {
|
||||
url: String,
|
||||
// Master often omits publicUrl when it matches url (Go json omitempty).
|
||||
#[serde(rename = "publicUrl", default)]
|
||||
#[serde(rename = "publicUrl")]
|
||||
public_url: String,
|
||||
#[serde(rename = "grpcPort", default)]
|
||||
grpc_port: u32,
|
||||
}
|
||||
|
||||
impl VolumeLocation {
|
||||
fn public_or_url(&self) -> &str {
|
||||
if self.public_url.is_empty() {
|
||||
&self.url
|
||||
} else {
|
||||
&self.public_url
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Master /dir/lookup response.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LookupResult {
|
||||
@@ -546,13 +514,9 @@ async fn do_replicated_request(
|
||||
.await
|
||||
.map_err(|e| format!("lookup volume failed: {}", e))?;
|
||||
|
||||
let self_http = to_http_address(&state.self_url);
|
||||
let remote_locations: Vec<_> = locations
|
||||
.into_iter()
|
||||
.filter(|loc| {
|
||||
to_http_address(&loc.url) != self_http
|
||||
&& to_http_address(loc.public_or_url()) != self_http
|
||||
})
|
||||
.filter(|loc| loc.url != state.self_url && loc.public_url != state.self_url)
|
||||
.collect();
|
||||
|
||||
if remote_locations.is_empty() {
|
||||
@@ -1488,16 +1452,12 @@ async fn get_or_head_handler_inner(
|
||||
if is_compressed {
|
||||
if needs_image_ops {
|
||||
// Always decompress for image operations (Go decompresses before resize/crop)
|
||||
match maybe_decompress_gzip(&data) {
|
||||
Ok(decompressed) => data = decompressed,
|
||||
Err(GunzipError::TooLarge) => {
|
||||
return (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"compressed object exceeds decompression limit",
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read as _;
|
||||
let mut decoder = GzDecoder::new(&data[..]);
|
||||
let mut decompressed = Vec::new();
|
||||
if decoder.read_to_end(&mut decompressed).is_ok() {
|
||||
data = decompressed;
|
||||
}
|
||||
} else {
|
||||
let accept_encoding = headers
|
||||
@@ -1514,16 +1474,12 @@ async fn get_or_head_handler_inner(
|
||||
response_headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
|
||||
} else {
|
||||
// Decompress for client
|
||||
match maybe_decompress_gzip(&data) {
|
||||
Ok(decompressed) => data = decompressed,
|
||||
Err(GunzipError::TooLarge) => {
|
||||
return (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"compressed object exceeds decompression limit",
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
Err(GunzipError::Decode) => {} // not valid gzip; keep raw bytes
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read as _;
|
||||
let mut decoder = GzDecoder::new(&data[..]);
|
||||
let mut decompressed = Vec::new();
|
||||
if decoder.read_to_end(&mut decompressed).is_ok() {
|
||||
data = decompressed;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2178,30 +2134,15 @@ pub async fn post_handler(
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
// Read body, bounded by the configured file-size limit so a single upload
|
||||
// cannot buffer unbounded memory and OOM-kill the server (mirrors Go's
|
||||
// io.LimitReader(r.Body, sizeLimit+1)). A margin covers multipart framing;
|
||||
// the exact per-file limit is still enforced on the parsed data below.
|
||||
let body_limit = if state.file_size_limit_bytes > 0 {
|
||||
// try_from (not `as usize`) so a >usize::MAX limit on 32-bit caps at
|
||||
// usize::MAX instead of silently truncating/wrapping to a tiny value.
|
||||
usize::try_from(state.file_size_limit_bytes)
|
||||
.unwrap_or(usize::MAX)
|
||||
.saturating_add(UPLOAD_BODY_OVERHEAD)
|
||||
} else {
|
||||
usize::MAX
|
||||
};
|
||||
let body = match axum::body::to_bytes(request.into_body(), body_limit).await {
|
||||
// Read body
|
||||
let body = match axum::body::to_bytes(request.into_body(), usize::MAX).await {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
// With a limit configured, an error here means the body exceeded it
|
||||
// before we buffered the whole thing; report it like the size check.
|
||||
let msg = if state.file_size_limit_bytes > 0 {
|
||||
format!("file over the limited {} bytes", state.file_size_limit_bytes)
|
||||
} else {
|
||||
format!("read body: {}", e)
|
||||
};
|
||||
return json_error_with_query(StatusCode::BAD_REQUEST, msg, Some(&query));
|
||||
return json_error_with_query(
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("read body: {}", e),
|
||||
Some(&query),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2341,17 +2282,7 @@ pub async fn post_handler(
|
||||
};
|
||||
|
||||
let uncompressed_data = if is_gzipped {
|
||||
match maybe_decompress_gzip(&body_data_raw) {
|
||||
Ok(d) => d,
|
||||
Err(GunzipError::TooLarge) => {
|
||||
return json_error_with_query(
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"compressed object exceeds decompression limit",
|
||||
Some(&query),
|
||||
);
|
||||
}
|
||||
Err(GunzipError::Decode) => body_data_raw.clone(),
|
||||
}
|
||||
maybe_decompress_gzip(&body_data_raw).unwrap_or_else(|| body_data_raw.clone())
|
||||
} else {
|
||||
body_data_raw.clone()
|
||||
};
|
||||
@@ -2830,16 +2761,14 @@ pub async fn delete_handler(
|
||||
// If this is a chunk manifest, delete child chunks first
|
||||
if n.is_chunk_manifest() {
|
||||
let manifest_data = if n.is_compressed() {
|
||||
match maybe_decompress_gzip(&n.data) {
|
||||
Ok(d) => d,
|
||||
Err(GunzipError::TooLarge) => {
|
||||
return json_error_with_query(
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"compressed manifest exceeds decompression limit",
|
||||
Some(&del_query),
|
||||
);
|
||||
}
|
||||
Err(GunzipError::Decode) => n.data.clone(),
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read as _;
|
||||
let mut decoder = GzDecoder::new(&n.data[..]);
|
||||
let mut decompressed = Vec::new();
|
||||
if decoder.read_to_end(&mut decompressed).is_ok() {
|
||||
decompressed
|
||||
} else {
|
||||
n.data.clone()
|
||||
}
|
||||
} else {
|
||||
n.data.clone()
|
||||
@@ -3195,19 +3124,14 @@ fn try_expand_chunk_manifest(
|
||||
last_modified_str: &Option<String>,
|
||||
) -> Option<Response> {
|
||||
let data = if n.is_compressed() {
|
||||
match maybe_decompress_gzip(&n.data) {
|
||||
Ok(d) => d,
|
||||
Err(GunzipError::TooLarge) => {
|
||||
return Some(
|
||||
(
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"compressed manifest exceeds decompression limit",
|
||||
)
|
||||
.into_response(),
|
||||
)
|
||||
}
|
||||
Err(GunzipError::Decode) => return None,
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read as _;
|
||||
let mut decoder = GzDecoder::new(&n.data[..]);
|
||||
let mut decompressed = Vec::new();
|
||||
if decoder.read_to_end(&mut decompressed).is_err() {
|
||||
return None;
|
||||
}
|
||||
decompressed
|
||||
} else {
|
||||
n.data.clone()
|
||||
};
|
||||
@@ -3217,13 +3141,6 @@ fn try_expand_chunk_manifest(
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
// Guard the attacker-controlled manifest size before allocating: a negative
|
||||
// value would wrap to a huge usize (capacity-overflow panic) and an oversized
|
||||
// one would OOM-kill the server.
|
||||
if manifest.size < 0 || manifest.size as u64 > MAX_EXPANSION_BYTES {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Read and concatenate all chunks
|
||||
let mut result = vec![0u8; manifest.size as usize];
|
||||
let store = state.store.read().unwrap();
|
||||
@@ -3257,54 +3174,24 @@ fn try_expand_chunk_manifest(
|
||||
)
|
||||
}
|
||||
}
|
||||
// Validate the attacker-controlled chunk offset before indexing: a
|
||||
// negative value would wrap to a huge usize, and an out-of-range one has
|
||||
// nowhere to land.
|
||||
if chunk.offset < 0 {
|
||||
return Some(
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("invalid negative chunk offset in {}", chunk.fid),
|
||||
)
|
||||
.into_response(),
|
||||
);
|
||||
}
|
||||
let offset = chunk.offset as usize;
|
||||
if offset >= result.len() {
|
||||
continue;
|
||||
}
|
||||
// Write the chunk into its window in `result`. Compressed chunks inflate
|
||||
// directly into the destination slice (bounded by the remaining window),
|
||||
// so a chunk never allocates a second large buffer — peak memory stays at
|
||||
// ~manifest.size instead of doubling. Bytes past the window are dropped,
|
||||
// matching the prior truncation behavior.
|
||||
if chunk_needle.is_compressed() {
|
||||
let chunk_data = if chunk_needle.is_compressed() {
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read as _;
|
||||
let mut decoder = GzDecoder::new(&chunk_needle.data[..]);
|
||||
let window = &mut result[offset..];
|
||||
let mut written = 0usize;
|
||||
let mut decode_failed = false;
|
||||
while written < window.len() {
|
||||
match decoder.read(&mut window[written..]) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => written += n,
|
||||
Err(_) => {
|
||||
decode_failed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// On any decode failure, drop the partial output and fall back to the
|
||||
// chunk's raw bytes (truncated to the window), preserving prior behavior.
|
||||
if decode_failed {
|
||||
window[..written].fill(0);
|
||||
let copy_len = chunk_needle.data.len().min(window.len());
|
||||
window[..copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
|
||||
let mut decompressed = Vec::new();
|
||||
if decoder.read_to_end(&mut decompressed).is_ok() {
|
||||
decompressed
|
||||
} else {
|
||||
chunk_needle.data.clone()
|
||||
}
|
||||
} else {
|
||||
let copy_len = chunk_needle.data.len().min(result.len() - offset);
|
||||
result[offset..offset + copy_len].copy_from_slice(&chunk_needle.data[..copy_len]);
|
||||
chunk_needle.data.clone()
|
||||
};
|
||||
let offset = chunk.offset as usize;
|
||||
let end = std::cmp::min(offset + chunk_data.len(), result.len());
|
||||
let copy_len = end - offset;
|
||||
if copy_len > 0 {
|
||||
result[offset..offset + copy_len].copy_from_slice(&chunk_data[..copy_len]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3677,22 +3564,13 @@ fn try_gzip_data(data: &[u8]) -> Option<Vec<u8>> {
|
||||
encoder.finish().ok()
|
||||
}
|
||||
|
||||
fn maybe_decompress_gzip(data: &[u8]) -> Result<Vec<u8>, GunzipError> {
|
||||
fn maybe_decompress_gzip(data: &[u8]) -> Option<Vec<u8>> {
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read;
|
||||
// Cap the output so a crafted, highly-compressible (gzip-bomb) needle cannot
|
||||
// OOM the server when decompressed on read or upload. take(limit+1) lets us
|
||||
// tell "exactly at the limit" apart from "over it", which we reject as
|
||||
// TooLarge so callers fail the request instead of silently using raw bytes.
|
||||
let mut decoder = GzDecoder::new(data).take(MAX_EXPANSION_BYTES + 1);
|
||||
let mut decoder = GzDecoder::new(data);
|
||||
let mut decompressed = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut decompressed)
|
||||
.map_err(|_| GunzipError::Decode)?;
|
||||
if decompressed.len() as u64 > MAX_EXPANSION_BYTES {
|
||||
return Err(GunzipError::TooLarge);
|
||||
}
|
||||
Ok(decompressed)
|
||||
decoder.read_to_end(&mut decompressed).ok()?;
|
||||
Some(decompressed)
|
||||
}
|
||||
|
||||
fn compute_md5_base64(data: &[u8]) -> String {
|
||||
@@ -3934,11 +3812,7 @@ mod tests {
|
||||
let compressed = try_gzip_data(data).unwrap();
|
||||
let decompressed = maybe_decompress_gzip(&compressed).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
// Non-gzip input is reported as a decode error (callers fall back to raw).
|
||||
assert!(matches!(
|
||||
maybe_decompress_gzip(data),
|
||||
Err(GunzipError::Decode)
|
||||
));
|
||||
assert!(maybe_decompress_gzip(data).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4037,26 +3911,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Master /dir/lookup often omits publicUrl when empty (Go `json:"publicUrl,omitempty"`).
|
||||
/// Replication must still parse locations or every cross-DC write fails.
|
||||
#[test]
|
||||
fn test_lookup_result_deserializes_without_public_url() {
|
||||
let body = r#"{
|
||||
"volumeOrFileId": "9",
|
||||
"locations": [
|
||||
{"url": "volume-a.example:8080", "dataCenter": "dc-a", "grpcPort": 18080},
|
||||
{"url": "volume-b.example:8080", "dataCenter": "dc-b", "grpcPort": 18080}
|
||||
]
|
||||
}"#;
|
||||
let result: LookupResult = serde_json::from_str(body).expect("parse lookup JSON");
|
||||
let locations = result.locations.expect("locations");
|
||||
assert_eq!(locations.len(), 2);
|
||||
assert_eq!(locations[0].url, "volume-a.example:8080");
|
||||
assert!(locations[0].public_url.is_empty());
|
||||
assert_eq!(locations[0].public_or_url(), "volume-a.example:8080");
|
||||
assert_eq!(locations[0].grpc_port, 18080);
|
||||
}
|
||||
|
||||
/// Regression test for issue #9274.
|
||||
///
|
||||
/// SeaweedFS encodes a server's gRPC port by appending `.grpcPort` to
|
||||
|
||||
@@ -4,10 +4,9 @@
|
||||
//! matching Go's `server/volume_grpc_client_to_master.go`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{error, info, warn};
|
||||
@@ -315,7 +314,6 @@ fn collect_ec_shard_delta_messages(
|
||||
disk_type: ec_vol.disk_type.to_string(),
|
||||
expire_at_sec: ec_vol.expire_at_sec,
|
||||
disk_id: disk_id as u32,
|
||||
encode_ts_ns: ec_vol.encode_ts_ns,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
@@ -828,27 +826,6 @@ fn build_heartbeat_with_ec_status(
|
||||
delete_vids.push(vol.id);
|
||||
should_delete_volume = true;
|
||||
} else if !vol.is_expired(volume_size, volume_size_limit) {
|
||||
// Detect phantom volumes: the .dat was unlinked from disk but is still
|
||||
// held open as a deleted FD, so the volume keeps serving and heartbeating
|
||||
// while no disk-path operation can ever succeed. Skip remote-tiered volumes,
|
||||
// whose .dat legitimately lives in cloud storage. Only a present .dat is
|
||||
// cached for 30s; a missing one is re-checked every heartbeat so the volume
|
||||
// stays suppressed until the file returns. See issues/10004
|
||||
if vol.file_count() > 0 && !vol.has_remote_file {
|
||||
const DISK_CHECK_INTERVAL_NS: i64 = 30 * 1_000_000_000;
|
||||
let now_ns = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_nanos() as i64;
|
||||
if now_ns - vol.last_disk_check_ns.load(Ordering::Relaxed) > DISK_CHECK_INTERVAL_NS {
|
||||
if !Path::new(&vol.file_name(".dat")).exists() {
|
||||
warn!("Volume {}: data file {} missing (held open as deleted FD) - not reporting to master", vol.id.0, vol.file_name(".dat"));
|
||||
continue;
|
||||
}
|
||||
vol.last_disk_check_ns.store(now_ns, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
let (remote_storage_name, remote_storage_key) = vol.remote_storage_name_key();
|
||||
volumes.push(master_pb::VolumeInformationMessage {
|
||||
id: vol.id.0,
|
||||
@@ -1069,7 +1046,6 @@ mod tests {
|
||||
write_queue: std::sync::OnceLock::new(),
|
||||
s3_tier_registry: std::sync::RwLock::new(S3TierRegistry::new()),
|
||||
read_mode: ReadMode::Local,
|
||||
allow_untrusted_remote_endpoints: false,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user