Compare commits

..
2261 changed files with 79316 additions and 280201 deletions
-22
View File
@@ -1,22 +0,0 @@
[codespell]
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
skip = .git,.git-meta,.gitignore,.gitattributes,*.svg,go.sum,vendor,*.lock,*.css,*.min.*,.codespellrc,*_templ.go
check-hidden = true
# Ignore camelCase and PascalCase identifiers (very common in Go/Rust/JS
# source, e.g. allLocations, publishErr, ReadInside, FlushInterval).
ignore-regex = \b[a-z]+[A-Z]\w*\b|\b[A-Z][a-z]+[A-Z]\w*\b
# visibles: variable name for VisibleInterval collections in filer/mount code
# fo: `*FilerOptions` receiver name (e.g. `func (fo *FilerOptions) ...`)
# te: "truncate error" local variable (e.g. `if te := w.Truncate(end); te != nil`)
# ser: Rust serde serializer variable (`serde_json::ser`, `let mut ser = ...`)
# bject: intentional wildcard test data (e.g. `s3:Get?bject` matching `s3:GetObject`)
# unparseable: accepted alternate spelling used throughout the codebase
# keep-alives: correct plural of the technical term (SSH/HTTP keep-alive)
# tread: valid English word in the idiom "tread carefully" (help text)
# anc: variable abbreviation for "ancestor" in tree/path tests
# ue: appears inside JSON test fixtures with embedded escaped quotes (Bl\"ue)
# auther: local variable meaning "authenticator" (tls.go: `auther := Authenticator{}`)
# thirdparty: literal Maven groupId `org.apache.hadoop.thirdparty` (external, cannot rename)
# unknwon: GitHub username / Go module path (`github.com/unknwon/goconfig`)
# atleast: CLI mode literal string in test/benchmark/fuse_db/bin/sqlite_verify.py
ignore-words-list = visibles,fo,te,ser,bject,unparseable,keep-alives,tread,anc,ue,auther,thirdparty,unknwon,atleast
@@ -1,66 +0,0 @@
name: Fix fusermount3 setuid
description: >
Make sure the fusermount3 an unprivileged mount will find can actually mount.
Some runner images carry a second, source-built fusermount3 in /usr/local/bin
that shadows the distro one in PATH; it is neither setuid nor root-owned, so
every mount fails with "mount failed: Operation not permitted". Both the Go
mount and sw-fuse resolve the helper through PATH.
Run it after the step that apt-installs fuse3.
runs:
using: composite
steps:
- name: Point PATH at a fusermount3 that can mount
shell: bash
run: |
set -euo pipefail
# setuid only grants root when root owns the file, and exec follows
# symlinks, so judge the target.
can_mount() {
[ -u "$1" ] && [ "$(stat -Lc %u "$1")" = 0 ]
}
bin=$(command -v fusermount3 || true)
if [ -z "$bin" ]; then
echo "no fusermount3 in PATH" >&2
exit 1
fi
if can_mount "$bin"; then
ls -l "$bin"
exit 0
fi
echo "$bin cannot mount unprivileged:"
ls -l "$bin"
# The distro fusermount3 is setuid root and is the one meant to be used.
# Reach it through a symlink earlier in PATH: exec resolves the link, so
# the target keeps its setuid bit, and nothing on the image is modified.
for distro in /usr/bin/fusermount3 /bin/fusermount3; do
if [ "$distro" != "$bin" ] && can_mount "$distro"; then
mkdir -p "$RUNNER_TEMP/fuse-bin"
ln -sf "$distro" "$RUNNER_TEMP/fuse-bin/fusermount3"
echo "$RUNNER_TEMP/fuse-bin" >> "$GITHUB_PATH"
echo "using $distro instead"
ls -l "$distro"
exit 0
fi
done
# No usable distro binary, so setting the bit is the only way out - but
# the target came from PATH: do that only for a root-owned system binary,
# never for anything else that happens to sit there.
case "$bin" in
/bin/*|/sbin/*|/usr/bin/*|/usr/sbin/*|/usr/local/bin/*|/usr/local/sbin/*) ;;
*) echo "refusing to setuid $bin: outside the system bin paths" >&2; exit 1 ;;
esac
if [ -L "$bin" ] || [ ! -f "$bin" ] || [ ! -x "$bin" ]; then
echo "refusing to setuid $bin: not a regular executable file" >&2
exit 1
fi
if [ "$(stat -c %u "$bin")" != 0 ] || [ "$(stat -c %g "$bin")" != 0 ]; then
echo "refusing to setuid $bin: not owned by root:root" >&2
exit 1
fi
sudo chmod u+s "$bin"
ls -l "$bin"
+2 -12
View File
@@ -3,15 +3,6 @@ name: "go: build dev binaries"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/binaries_dev.yml'
concurrency:
group: binaries-dev-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -27,7 +18,6 @@ jobs:
- name: Delete old release assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
@@ -48,7 +38,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
@@ -97,7 +87,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+4 -9
View File
@@ -2,11 +2,6 @@ name: "Code Scanning - Action"
on:
pull_request:
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- '.github/workflows/codeql.yml'
concurrency:
group: ${{ github.head_ref }}/codeql
@@ -23,11 +18,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4.37.4
uses: github/codeql-action/init@v4
# Override language selection by uncommenting this and choosing your languages
with:
languages: go
@@ -35,7 +30,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below).
- name: Autobuild
uses: github/codeql-action/autobuild@v4.37.4
uses: github/codeql-action/autobuild@v4
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -49,4 +44,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4.37.4
uses: github/codeql-action/analyze@v4
-23
View File
@@ -1,23 +0,0 @@
# Codespell configuration is within .codespellrc
---
name: Codespell
on:
push:
branches: [master]
pull_request:
branches: [master]
permissions:
contents: read
jobs:
codespell:
name: Check for spelling errors
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Codespell
uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
+23 -109
View File
@@ -3,109 +3,23 @@ name: "docker: build dev containers"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'seaweed-volume/**'
- 'docker/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/container_dev.yml'
permissions:
contents: read
jobs:
# ── Pre-build Rust volume server binaries natively ──────────────────
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-dev-${{ matrix.target }}-
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: weed-volume-normal-${{ matrix.arch }}
build-dev-containers:
needs: [build-rust-binaries]
runs-on: [ubuntu-latest]
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Download pre-built Rust binaries
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-normal-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Docker meta
-
name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
with:
images: |
chrislusf/seaweedfs
@@ -116,40 +30,40 @@ jobs:
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
uses: docker/setup-qemu-action@v4.2.0
- name: Create BuildKit config
-
name: Set up QEMU
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
-
name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build
uses: docker/build-push-action@v7
-
name: Build
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
@@ -58,7 +58,7 @@ jobs:
sudo ldconfig
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: go.mod
@@ -126,31 +126,27 @@ jobs:
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v4.2.0
uses: docker/setup-qemu-action@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Determine branch to build
id: branch
env:
INPUT_REF: ${{ inputs.seaweedfs_ref }}
HEAD_REF: ${{ github.head_ref }}
REF_NAME: ${{ github.ref_name }}
run: |
if [ -n "$INPUT_REF" ]; then
echo "branch=$INPUT_REF" >> "$GITHUB_OUTPUT"
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
elif [ "${{ github.event_name }}" = "pull_request" ]; then
echo "branch=$HEAD_REF" >> "$GITHUB_OUTPUT"
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
else
echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT"
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push image
+19 -136
View File
@@ -1,10 +1,9 @@
name: "docker: build latest container"
# Manual fallback only. On tag push, container_release_unified.yml already
# re-tags the released versioned image as `latest` / `latest_large_disk`,
# so a full rebuild here is unnecessary. Run this manually if you need to
# rebuild `latest` from an arbitrary ref.
on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
source_ref:
@@ -59,80 +58,8 @@ jobs:
echo "publish=true" >> "$GITHUB_OUTPUT"
fi
# ── Pre-build Rust volume server binaries natively ──────────────────
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v7
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: |
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
build:
needs: [setup, build-rust-binaries]
needs: [setup]
runs-on: ubuntu-latest
strategy:
matrix:
@@ -141,7 +68,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Free Disk Space
@@ -167,32 +94,11 @@ jobs:
if [ "${{ matrix.variant }}" == "large_disk" ]; then
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
else
echo "tag_suffix=" >> $GITHUB_OUTPUT
echo "build_args=" >> $GITHUB_OUTPUT
echo "rust_variant=normal" >> $GITHUB_OUTPUT
fi
- name: Download pre-built Rust binaries
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
@@ -207,7 +113,7 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: matrix.platform != 'amd64'
uses: docker/setup-qemu-action@v4.2.0
uses: docker/setup-qemu-action@v4
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
@@ -221,13 +127,13 @@ jobs:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
@@ -260,7 +166,7 @@ jobs:
trivy-scan:
runs-on: ubuntu-latest
needs: [setup, build, build-rust-binaries]
needs: [setup, build]
strategy:
matrix:
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
@@ -275,39 +181,16 @@ jobs:
fi
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Checkout for local scan build
if: needs.setup.outputs.publish != 'true'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Download pre-built Rust binaries for local scan
if: needs.setup.outputs.publish != 'true'
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context for local scan
if: needs.setup.outputs.publish != 'true'
run: |
rust_variant="normal"
if [ "${{ matrix.variant }}" == "large_disk" ]; then
rust_variant="large-disk"
fi
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Create BuildKit config for local scan build
if: needs.setup.outputs.publish != 'true'
run: |
@@ -338,7 +221,7 @@ jobs:
- name: Trivy report (published image)
if: needs.setup.outputs.publish == 'true'
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
scan-type: image
# Scan amd64 only - OS packages are identical across architectures
@@ -355,7 +238,7 @@ jobs:
exit-code: '0'
- name: Trivy report (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
@@ -367,7 +250,7 @@ jobs:
output: trivy-results.sarif
exit-code: '0'
- name: Upload Trivy scan results to GitHub Security
uses: github/codeql-action/upload-sarif@v4.37.4
uses: github/codeql-action/upload-sarif@v4
if: always()
with:
sarif_file: trivy-results.sarif
@@ -375,7 +258,7 @@ jobs:
if: needs.setup.outputs.publish == 'true'
# Gate only on fixable high/critical vulnerabilities. Non-fixable
# findings are still visible in the SARIF upload above.
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
scan-type: image
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
@@ -388,7 +271,7 @@ jobs:
skip-setup-trivy: true
- name: Trivy gate (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
@@ -408,7 +291,7 @@ jobs:
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
@@ -430,12 +313,12 @@ jobs:
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
- name: Login to Docker Hub
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
@@ -4,19 +4,11 @@ on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag to publish (e.g. 3.93)'
required: true
default: ''
workflow_dispatch: {}
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
jobs:
build-large-release-container_foundationdb:
@@ -25,7 +17,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
-
name: Docker meta
id: docker_meta
@@ -34,7 +26,7 @@ jobs:
images: |
chrislusf/seaweedfs
tags: |
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
type=ref,event=tag,suffix=_large_disk_foundationdb
flavor: |
latest=false
labels: |
@@ -43,14 +35,14 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@v4.2.0
uses: docker/setup-qemu-action@v4
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
+109 -326
View File
@@ -29,11 +29,9 @@ on:
permissions:
contents: read
security-events: write
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
IMAGE: ghcr.io/chrislusf/seaweedfs
# Limit concurrent builds to avoid rate limits
concurrency:
@@ -41,168 +39,86 @@ concurrency:
cancel-in-progress: false
jobs:
# ── Pre-build Rust volume server binaries natively ──────────────────
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
# emulated cargo build into ~15 minutes of native compilation.
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: |
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
# One job per (variant, platform) on a native runner, pushed by digest;
# the merge job stitches the digests into one multi-arch tag.
build:
needs: [build-rust-binaries]
runs-on: ${{ matrix.runner }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
# Build sequentially to avoid rate limits
max-parallel: 2
matrix:
include:
# Normal volume - multi-arch
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
# Large disk - multi-arch
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
- variant: normal
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: ""
tag_suffix: ""
# Large disk - multi-arch
- variant: large_disk
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset
tag_suffix: _large_disk
# Full tags - multi-arch
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
- variant: full
platforms: linux/amd64,linux/arm64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _full
# Large disk + full tags - multi-arch
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
- variant: large_disk_full
platforms: linux/amd64,linux/arm64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _large_disk_full
# RocksDB large disk - amd64 only
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- variant: rocksdb
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.rocksdb_large
build_args: ""
tag_suffix: _large_disk_rocksdb
steps:
- name: Skip unselected variant
if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant
run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0
- name: Checkout
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/checkout@v7
- name: Download pre-built Rust binaries
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
uses: actions/checkout@v6
- name: Free Disk Space
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
echo "Available disk space before cleanup:"
df -h
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
sudo docker system prune -af --volumes
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
id: docker_meta
uses: docker/metadata-action@v6
with:
images: ${{ env.IMAGE }}
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
flavor: latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu
uses: docker/setup-qemu-action@v4.2.0
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
uses: docker/setup-qemu-action@v4
- name: Create BuildKit config
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
@@ -210,111 +126,91 @@ jobs:
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/setup-buildx-action@v4
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.6.0
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build and push ${{ matrix.variant }} (${{ matrix.platform }})
- name: Build and push ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
id: build
uses: docker/build-push-action@v7
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platform }}
platforms: ${{ matrix.platforms }}
# Push to GHCR to avoid Docker Hub rate limits on pulls
tags: |
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
labels: ${{ steps.docker_meta.outputs.labels }}
# Flat single-platform manifest so imagetools create assembles cleanly.
provenance: false
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
# max only for rocksdb: its RocksDB compile is sha-independent and worth
# keeping; go-build layers are sha-busted every release, so min elsewhere.
cache-to: type=gha,mode=${{ matrix.variant == 'rocksdb' && 'max' || 'min' }},scope=${{ matrix.variant }}-${{ matrix.arch }}
cache-from: type=gha,scope=${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
build-args: |
${{ matrix.build_args }}
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
- name: Export digest
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
- name: Clean up build artifacts
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"
sudo docker system prune -f
sudo rm -rf /tmp/go-build*
- name: Upload digest
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/upload-artifact@v7
with:
name: digest-${{ matrix.variant }}-${{ matrix.arch }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
# Assemble each variant's per-platform digests into one tag, then mirror to Docker Hub.
merge:
needs: [build]
copy-to-dockerhub:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
strategy:
fail-fast: false
matrix:
variant: [normal, large_disk, full, large_disk_full, rocksdb]
include:
- { variant: normal, tag_suffix: "" }
- { variant: large_disk, tag_suffix: _large_disk }
- { variant: full, tag_suffix: _full }
- { variant: large_disk_full, tag_suffix: _large_disk_full }
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb }
- variant: normal
tag_suffix: ""
- variant: large_disk
tag_suffix: _large_disk
- variant: full
tag_suffix: _full
- variant: large_disk_full
tag_suffix: _large_disk_full
- variant: rocksdb
tag_suffix: _large_disk_rocksdb
steps:
- name: Download digests
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/download-artifact@v8
uses: docker/login-action@v4
with:
pattern: digest-${{ matrix.variant }}-*
merge-multiple: true
path: /tmp/digests
- name: Set up Docker Buildx
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/setup-buildx-action@v4
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.6.0
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Create multi-arch tag on GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
-t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.6.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Install crane
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
@@ -322,171 +218,58 @@ jobs:
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Copy ${{ matrix.variant }} to Docker Hub
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
# Function to retry with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Copy multi-arch image from GHCR to Docker Hub with retry
# This is much more efficient than pulling/pushing individual arch images
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
retry_with_backoff crane copy \
${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
echo "Copied ${{ matrix.variant }} to Docker Hub"
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
# Security for visibility, but never blocks the release. Releases (including
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
# we sometimes need to publish through known findings (e.g. unfixed upstream
# CVE, base-image lag).
trivy-scan:
runs-on: ubuntu-latest
needs: [merge]
if: github.event_name == 'push'
continue-on-error: true
strategy:
fail-fast: false
matrix:
include:
- source_suffix: ""
variant: normal
- source_suffix: _large_disk
variant: large_disk
steps:
- name: Login to GHCR
uses: docker/login-action@v4.6.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Trivy report (${{ matrix.variant }})
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: image
# Scan the multi-arch tag on GHCR (already pushed by the build job).
# Trivy scans the runner's native platform; OS packages are identical
# across architectures since they all share the same alpine base.
image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Upload Trivy scan results to GitHub Security
if: always()
uses: github/codeql-action/upload-sarif@v4.37.4
with:
sarif_file: trivy-results.sarif
category: trivy-${{ matrix.variant }}
# Point `latest` (and `latest_large_disk`) at the just-released versioned
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
# no QEMU, no separate workflow. Replaces the old container_latest.yml
# rebuild that often failed or lagged behind the release. Independent of
# trivy-scan: vuln findings are reported but do not block `latest`.
tag-latest:
runs-on: ubuntu-latest
needs: [merge]
if: github.event_name == 'push'
strategy:
matrix:
include:
- source_suffix: ""
latest_tag: latest
- source_suffix: _large_disk
latest_tag: latest_large_disk
steps:
- name: Login to Docker Hub
uses: docker/login-action@v4.6.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4.6.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
run: |
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
run: |
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
DST_TAG="${{ matrix.latest_tag }}"
echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}"
retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}"
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
helm-release:
runs-on: ubuntu-latest
needs: [build]
needs: [copy-to-dockerhub]
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
permissions:
contents: write
pages: write
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Prepare Docker tag
id: tag
@@ -82,19 +82,19 @@ jobs:
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v1
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v1
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push image
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
with:
context: ./docker
push: true
+4 -7
View File
@@ -21,22 +21,19 @@ jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'telemetry/server/go.mod'
go-version-file: 'go.mod'
- name: Build Telemetry Server
if: github.event_name == 'workflow_dispatch' && inputs.deploy
run: |
# telemetry/server is its own Go module; build from within it
cd telemetry/server
go mod tidy
echo "Building telemetry server..."
GOOS=linux GOARCH=amd64 go build -o ../../telemetry-server .
cd ../..
GOOS=linux GOARCH=amd64 go build -o telemetry-server ./telemetry/server/main.go
ls -la telemetry-server
echo "Build completed successfully"
+2 -2
View File
@@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- name: 'Dependency Review'
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
+6 -31
View File
@@ -3,20 +3,8 @@ name: "End to End"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'docker/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/e2e.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'docker/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/e2e.yml'
concurrency:
group: ${{ github.head_ref }}/e2e
@@ -36,23 +24,18 @@ jobs:
timeout-minutes: 30
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
@@ -74,12 +57,12 @@ jobs:
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
- name: Start SeaweedFS
timeout-minutes: 15
timeout-minutes: 10
run: |
# Enable Docker buildkit for better caching
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
# Build with retry logic
for i in {1..3}; do
echo "Build attempt $i/3"
@@ -94,18 +77,10 @@ jobs:
sleep 30
fi
done
# Start services with wait
docker compose -f ./compose/e2e-mount.yml up --wait
- name: Rotate buildx cache
if: always()
run: |
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
rm -rf /tmp/.buildx-cache
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
- name: Run FIO 4k
timeout-minutes: 15
run: |
+2 -14
View File
@@ -3,20 +3,8 @@ name: "EC Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/erasure_coding/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/ec-integration-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/erasure_coding/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/ec-integration-tests.yml'
permissions:
contents: read
@@ -28,13 +16,13 @@ jobs:
timeout-minutes: 30
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Build weed binary
run: |
+49
View File
@@ -0,0 +1,49 @@
name: EC Integration Tests
on:
push:
branches: [ master ]
paths:
- 'weed/admin/**'
- 'weed/worker/**'
- 'test/erasure_coding/admin_dockertest/**'
- '.github/workflows/ec-integration.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/admin/**'
- 'weed/worker/**'
- 'test/erasure_coding/admin_dockertest/**'
- '.github/workflows/ec-integration.yml'
jobs:
ec-integration-test:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build weed binary
run: |
cd weed
go build -o ../weed_bin
- name: Run EC integration tests
run: |
cd test/erasure_coding/admin_dockertest
go test -v -timeout 15m ec_integration_test.go
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: ec-test-logs
path: test/erasure_coding/admin_dockertest/tmp/logs/
retention-days: 7
@@ -1,68 +0,0 @@
name: "FUSE DLM Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
- '.github/workflows/fuse-dlm-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-dlm-integration:
name: FUSE DLM Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run DLM integration tests
timeout-minutes: 25
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-dlm-test-logs
path: /tmp/seaweedfs-fuse-dlm-logs/
retention-days: 3
+2 -7
View File
@@ -7,14 +7,12 @@ on:
- 'weed/**'
- 'test/fuse_integration/**'
- '.github/workflows/fuse-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
pull_request:
branches: [ master, main ]
paths:
- 'weed/**'
- 'test/fuse_integration/**'
- '.github/workflows/fuse-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.head_ref }}/fuse-integration
@@ -31,10 +29,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -51,9 +49,6 @@ jobs:
fusermount3 --version || fusermount --version || true
ls -la /dev/fuse
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
- name: Build SeaweedFS
run: |
cd weed
@@ -1,74 +0,0 @@
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
- '.github/workflows/fuse-p2p-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-p2p-integration:
name: FUSE P2P Peer Chunk Sharing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run P2P integration tests
timeout-minutes: 15
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-p2p-test-logs
path: /tmp/seaweedfs-fuse-p2p-logs/
retention-days: 3
+6 -80
View File
@@ -3,18 +3,8 @@ name: "go: build binary"
on:
push:
branches: [ master ]
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- '.github/workflows/go.yml'
pull_request:
branches: [ master ]
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- '.github/workflows/go.yml'
concurrency:
group: ${{ github.head_ref }}/go
@@ -30,9 +20,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Get dependencies
@@ -47,92 +37,28 @@ jobs:
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
if grep -q "vet:" vet-output.txt; then exit 1; fi
vet-32bit:
name: Go Vet 32-bit
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
run: |
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
build-cross:
name: Build cross-platform (${{ matrix.goos }}/${{ matrix.goarch }})
runs-on: ubuntu-latest
strategy:
# One target's breakage should not hide the other three.
fail-fast: false
matrix:
include:
- { goos: windows, goarch: amd64 }
- { goos: windows, goarch: arm64 }
- { goos: freebsd, goarch: amd64 }
- { goos: darwin, goarch: arm64 }
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# Nothing else on a PR compiles these, so per-OS syscall constants creep
# back into shared files unnoticed and only a release build catches it.
- name: Cross-compile
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
run: |
go build ./weed/...
# Tests too: they reach for per-OS syscall constants the build does
# not, and only compiling them catches an untagged one.
go vet ./weed/mount/... ./weed/command/... 2>&1 |
grep -v "MessageState contains sync.Mutex" | tee /tmp/vet.txt
if grep -q "vet:" /tmp/vet.txt; then exit 1; fi
test:
name: Test
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Test
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
test-32bit:
name: Test 32-bit
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# 386 test binaries run natively on the amd64 runner. This catches what vet
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
# -short skips the e2e suites already covered on amd64.
- name: Test linux/386
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
File diff suppressed because it is too large Load Diff
+2 -5
View File
@@ -1,9 +1,6 @@
name: "helm: release"
name: "helm: manual release"
on:
push:
tags:
- '*'
workflow_dispatch:
permissions:
@@ -15,7 +12,7 @@ jobs:
helm-release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: 0
+2 -2
View File
@@ -25,10 +25,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
-183
View File
@@ -1,183 +0,0 @@
name: "release: java clients"
# Publishes the SeaweedFS Java clients (seaweedfs-client and
# seaweedfs-hadoop3-client) to Maven Central via the Sonatype Central Portal.
#
# Required repository secrets:
# MAVEN_CENTRAL_USERNAME - Central Portal user token username (central.sonatype.com -> Account -> Generate User Token)
# MAVEN_CENTRAL_PASSWORD - Central Portal user token password
# MAVEN_GPG_PRIVATE_KEY - ASCII-armored signing key:
# gpg --homedir <keyring> --armor --export-secret-keys <KEYID> > key.asc
# MAVEN_GPG_PASSPHRASE - passphrase for that key
#
# A plain run publishes to Central. Check dry_run to exercise secrets, GPG
# signing, and the build without committing or publishing.
on:
workflow_dispatch:
inputs:
version:
description: "Version to publish, e.g. 4.39. Leave blank to use the current SeaweedFS version."
type: string
required: false
dry_run:
description: "Build and GPG-sign only; skip the version commit and the Central upload"
type: boolean
default: false
permissions:
contents: write
concurrency:
group: java-release
cancel-in-progress: false
env:
# Java 17 needs these opens for the Maven publishing plugins' reflective access.
MAVEN_OPTS: >-
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.text=ALL-UNNAMED
--add-opens=java.desktop/java.awt.font=ALL-UNNAMED
jobs:
publish:
name: Publish to Maven Central
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
# Full history so the version-bump commit can rebase onto a moved master.
fetch-depth: 0
# An explicit input wins; otherwise fall back to the current SeaweedFS
# version (MAJOR.MINOR) from constants.go, matching its %d.%02d formatting.
- name: Resolve version
id: resolve
env:
INPUT_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
VERSION="$INPUT_VERSION"
if [ -z "$VERSION" ]; then
CONST=weed/util/version/constants.go
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
echo "No version given; using current SeaweedFS version ${VERSION}."
fi
if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
echo "::error::version must be MAJOR.MINOR, e.g. 4.39 (got '$VERSION')"
exit 1
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
- name: Set up JDK 17
uses: actions/setup-java@v5
with:
java-version: '17'
distribution: 'temurin'
cache: 'maven'
server-id: central
server-username: MAVEN_CENTRAL_USERNAME
server-password: MAVEN_CENTRAL_PASSWORD
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
# client carries the version literally; hdfs3 derives its own version and
# its seaweedfs-client dependency from the seaweedfs.client.version property.
- name: Set versions in poms
env:
VERSION: ${{ steps.resolve.outputs.version }}
run: |
set -euo pipefail
V=org.codehaus.mojo:versions-maven-plugin:2.16.2
mvn -B -ntp -f other/java/client/pom.xml "${V}:set" \
-DnewVersion="$VERSION" -DgenerateBackupPoms=false
mvn -B -ntp -f other/java/hdfs3/pom.xml "${V}:set-property" \
-Dproperty=seaweedfs.client.version -DnewVersion="$VERSION" -DgenerateBackupPoms=false
- name: Commit version bump
if: ${{ !inputs.dry_run }}
env:
VERSION: ${{ steps.resolve.outputs.version }}
run: |
set -euo pipefail
if git diff --quiet; then
echo "Poms already at ${VERSION}; nothing to commit."
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add other/java/client/pom.xml other/java/hdfs3/pom.xml
git commit -m "java ${VERSION}"
# Rebase and retry so a concurrent push to master doesn't lose the bump.
for attempt in 1 2 3 4 5; do
if git push origin HEAD:master; then
exit 0
fi
echo "push rejected (attempt ${attempt}); rebasing onto latest master"
git pull --rebase origin master
done
echo "::error::could not push version bump after retries"
exit 1
# Tests are skipped here: the integration tests need a live filer, and the
# offline unit tests already run on every push in java_unit_tests.yml.
# dry_run stops at `install` (build + GPG sign, no upload). A real run deploys.
# client goes first either way so its install populates the local repo for hdfs3.
- name: Publish seaweedfs-client
working-directory: other/java/client
env:
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
DRY_RUN: ${{ inputs.dry_run }}
run: |
set -euo pipefail
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
mvn -B -ntp $GOAL -DskipTests
- name: Publish seaweedfs-hadoop3-client
working-directory: other/java/hdfs3
env:
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
DRY_RUN: ${{ inputs.dry_run }}
run: |
set -euo pipefail
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
mvn -B -ntp $GOAL -DskipTests
# Bump every version the wiki quotes for the Java clients. Runs only after
# a successful real publish. Uses GITHUB_TOKEN; set a WIKI_TOKEN secret if
# the token can't push to the wiki.
- name: Update wiki client versions
if: ${{ !inputs.dry_run }}
env:
VERSION: ${{ steps.resolve.outputs.version }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WIKI_TOKEN: ${{ secrets.WIKI_TOKEN }}
run: |
set -euo pipefail
TOKEN="${WIKI_TOKEN:-$GH_TOKEN}"
git clone -q "https://x-access-token:${TOKEN}@github.com/${{ github.repository }}.wiki.git" wiki
cd wiki
for f in $(grep -rlE "seaweedfs-(client|hadoop3-client)" --include="*.md" .); do
NEWV="$VERSION" perl -0777 -i -pe '
my $v = $ENV{NEWV};
s{(<artifactId>seaweedfs-(?:client|hadoop3-client)</artifactId>\s*<version>)[0-9]+\.[0-9]+(</version>)}{$1$v$2}g;
s{(seaweedfs-(?:client|hadoop3-client):)[0-9]+\.[0-9]+}{$1$v}g;
s{(seaweedfs-(?:client|hadoop3-client)-)[0-9]+\.[0-9]+(\.jar)}{$1$v$2}g;
s{(seaweedfs-(?:client|hadoop3-client)/)[0-9]+\.[0-9]+(/)}{$1$v$2}g;
' "$f"
done
if git diff --quiet; then
echo "Wiki already references ${VERSION}."
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git commit -am "java clients ${VERSION}"
git push
+1 -1
View File
@@ -23,7 +23,7 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Java
uses: actions/setup-java@v5
+2 -23
View File
@@ -3,24 +3,8 @@ name: "Kafka Quick Test (Load Test with Schema Registry)"
on:
push:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-quicktest.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-quicktest.yml'
workflow_dispatch: # Allow manual trigger
concurrency:
@@ -37,10 +21,10 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
@@ -48,11 +32,6 @@ jobs:
**/go.sum
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
+15 -31
View File
@@ -3,24 +3,8 @@ name: "Kafka Gateway Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-tests.yml'
concurrency:
group: ${{ github.head_ref }}/kafka-tests
@@ -51,13 +35,13 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Setup Container Environment
run: |
@@ -96,13 +80,13 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Setup Integration Container Environment
run: |
@@ -143,10 +127,10 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
@@ -322,10 +306,10 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
@@ -461,7 +445,7 @@ jobs:
# Test consumer group functionality with explicit timeout
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
ulimit -u 100 || echo "Warning: Could not set process limit"
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
env:
GOMAXPROCS: 1
SEAWEEDFS_MASTERS: 127.0.0.1:9333
@@ -484,10 +468,10 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
@@ -642,10 +626,10 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
@@ -803,13 +787,13 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Setup Protocol Container Environment
run: |
+4 -4
View File
@@ -37,10 +37,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -82,10 +82,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -1,79 +0,0 @@
name: "Master Cold Start Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/topology/**'
- 'weed/operation/**'
- 'test/master_cold_start/**'
- 'test/testutil/**'
- '.github/workflows/master-cold-start-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/topology/**'
- 'weed/operation/**'
- 'test/master_cold_start/**'
- 'test/testutil/**'
- '.github/workflows/master-cold-start-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
master-cold-start-tests:
name: Master Cold Start Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v7
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build weed binary
run: |
cd weed && go install -buildvcs=false
- name: Run master cold start tests
# test/master_cold_start boots a fresh master plus empty volume
# servers and requires the very first assign (HTTP and gRPC, no
# client retries) to complete a write: the assign that triggers
# volume growth must wait for it instead of failing with
# "volume growth in progress".
run: |
export WEED_BINARY=$(go env GOPATH)/bin/weed
go test -v -timeout=8m ./test/master_cold_start/...
- name: Collect server logs on failure
if: failure()
run: |
# test/master_cold_start/cluster.go keeps failing-test dirs created
# via os.MkdirTemp("", "seaweedfs_master_cold_start_it_") with each
# process log under <baseDir>/logs/.
mkdir -p /tmp/master-cold-start-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_master_cold_start_it_*" 2>/dev/null | while read dir; do
echo "Found test directory: $dir"
cp -r "$dir" /tmp/master-cold-start-logs/ 2>/dev/null || true
done
find /tmp/master-cold-start-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: master-cold-start-test-logs
path: /tmp/master-cold-start-logs/
retention-days: 7
@@ -40,10 +40,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -1,124 +0,0 @@
name: "mount: windows conformance"
on:
push:
branches: [ master ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp-conformance/**'
- '.github/workflows/mount-windows-conformance.yml'
# No base branch filter: this is the only thing that runs the Windows mount,
# so it should cover a pull request stacked on another one too.
pull_request:
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp-conformance/**'
- '.github/workflows/mount-windows-conformance.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
jobs:
conformance:
name: WinFsp conformance
runs-on: windows-latest
timeout-minutes: 60
env:
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
CGO_ENABLED: 0
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
- name: Install WinFsp
run: choco install winfsp -y --no-progress
- name: Build weed.exe
run: go build -o weed.exe ./weed
# The runner tears down a step's process tree when its shell exits, so a
# cluster started in one step is gone by the next. Everything that needs
# the cluster and the mount alive has to share a step.
- name: Mount and run winfsp-tests
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
function Test-Port($port) {
# A plain connect, because Test-NetConnection has reported success
# here for a port nothing was listening on.
$client = New-Object System.Net.Sockets.TcpClient
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
catch { return $false }
finally { $client.Dispose() }
}
function Start-Mount($log) {
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
$deadline = (Get-Date).AddMinutes(2)
while ((Get-Date) -lt $deadline) {
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
Start-Sleep -Seconds 2
}
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
throw "S: never appeared"
}
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
# -ip pins the cluster to loopback; it otherwise advertises and binds
# the runner's LAN address, which 127.0.0.1 cannot reach.
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
$deadline = (Get-Date).AddMinutes(3)
while ((Get-Date) -lt $deadline) {
# The mount dials grpc, not http, so both ports have to answer.
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
Start-Sleep -Seconds 3
}
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
throw "filer never came up"
}
Write-Host "filer is up on http 8888 and grpc 18888"
Start-Mount 'seaweed-mount'
Write-Host "::group::winfsp-tests"
& pwsh -File test/winfsp-conformance/run.ps1 -MountPoint S:\
$code = $LASTEXITCODE
Write-Host "::endgroup::"
if ($code -ne 0) { throw "winfsp-tests failed with exit $code" }
- name: Logs
if: always()
shell: pwsh
run: |
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
}
-194
View File
@@ -1,194 +0,0 @@
name: "mount: windows"
on:
push:
branches: [ master ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp/**'
- '.github/workflows/mount-windows.yml'
# No base branch filter: this is the only thing that runs the Windows mount,
# so it should cover a pull request stacked on another one too.
pull_request:
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp/**'
- '.github/workflows/mount-windows.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
jobs:
mount-windows:
name: Mount on Windows
runs-on: windows-latest
timeout-minutes: 40
env:
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
CGO_ENABLED: 0
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
- name: Install WinFsp
run: choco install winfsp -y --no-progress
- name: Build weed.exe
run: go build -o weed.exe ./weed
# The runner tears down a step's process tree when its shell exits, so a
# cluster started in one step is gone by the next. Everything that needs
# the cluster and the mount alive has to share a step.
- name: Mount and exercise
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
function Test-Port($port) {
# A plain connect, because Test-NetConnection has reported success
# here for a port nothing was listening on.
$client = New-Object System.Net.Sockets.TcpClient
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
catch { return $false }
finally { $client.Dispose() }
}
function Start-Mount($log) {
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
$deadline = (Get-Date).AddMinutes(2)
while ((Get-Date) -lt $deadline) {
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
Start-Sleep -Seconds 2
}
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
throw "S: never appeared"
}
function Stop-Mount {
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
Where-Object { $_.CommandLine -like '*mount*' } |
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
$deadline = (Get-Date).AddMinutes(1)
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
if (Test-Path S:\) { throw "S: still present after stopping the mount" }
Write-Host "unmounted"
}
function Invoke-Tests($label, [string[]]$goArgs) {
Write-Host "::group::$label"
& go @goArgs
$code = $LASTEXITCODE
Write-Host "::endgroup::"
if ($code -ne 0) { throw "$label failed with exit $code" }
}
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
# -ip pins the cluster to loopback; it otherwise advertises and binds
# the runner's LAN address, which 127.0.0.1 cannot reach.
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
$deadline = (Get-Date).AddMinutes(3)
while ((Get-Date) -lt $deadline) {
# The mount dials grpc, not http, so both ports have to answer.
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
Start-Sleep -Seconds 3
}
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
throw "filer never came up"
}
Write-Host "filer is up on http 8888 and grpc 18888"
Start-Mount 'seaweed-mount'
Invoke-Tests 'exercise' @('test','-v','-timeout','20m','./test/winfsp','-mountpoint=S:\')
Invoke-Tests 'persist-write' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=write','-filer=127.0.0.1:8888')
Stop-Mount
Start-Mount 'seaweed-remount'
Invoke-Tests 'persist-verify' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=verify')
# WinFsp creates the mount directory itself, so the path must not
# exist; only its parent has to.
Write-Host "::group::mount over a directory"
Stop-Mount
Remove-Item C:\seaweed-mnt -Recurse -Force -ErrorAction SilentlyContinue
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=C:\seaweed-mnt' `
-RedirectStandardOutput C:\seaweed-dirmount.log -RedirectStandardError C:\seaweed-dirmount.err.log
$deadline = (Get-Date).AddMinutes(2)
$ok = $false
while ((Get-Date) -lt $deadline) {
# Listing succeeds on the plain empty directory too, so wait for the
# reparse point WinFsp turns it into. Otherwise this step passes
# without a mount and writes to local disk.
$item = Get-Item C:\seaweed-mnt -Force -ErrorAction SilentlyContinue
if ($null -ne $item -and $item.Attributes.ToString() -like '*ReparsePoint*') { $ok = $true; break }
Start-Sleep -Seconds 2
}
if (-not $ok) {
Get-Content C:\seaweed-dirmount.log, C:\seaweed-dirmount.err.log -ErrorAction SilentlyContinue
throw "mounting over a directory failed"
}
Set-Content -Path C:\seaweed-mnt\dirmount.txt -Value 'via directory mount'
if ((Get-Content C:\seaweed-mnt\dirmount.txt) -ne 'via directory mount') { throw "readback through the directory mount differs" }
Remove-Item C:\seaweed-mnt\dirmount.txt -Force
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
Where-Object { $_.CommandLine -like '*mount*' } |
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
Start-Sleep -Seconds 5
Write-Host "::endgroup::"
Start-Mount 'seaweed-remount2'
Write-Host "::group::explorer-style walk"
New-Item -ItemType Directory -Force -Path S:\walk | Out-Null
1..200 | ForEach-Object { Set-Content -Path "S:\walk\f$_.txt" -Value "line $_" }
$names = @(Get-ChildItem S:\walk | ForEach-Object { $_.Name })
if ($names.Count -ne 200) {
# Name the strays: a dot entry surfacing here is a different problem
# from a missing or duplicated file.
$unexpected = $names | Where-Object { $_ -notmatch '^f\d+\.txt$' }
throw "listed $($names.Count) entries, expected 200; unexpected: $($unexpected -join ', ')"
}
$body = Get-Content S:\walk\f42.txt
if ($body -ne 'line 42') { throw "unexpected content: $body" }
Copy-Item S:\walk\f42.txt S:\walk\copy.txt
Remove-Item S:\walk -Recurse -Force
if (Test-Path S:\walk) { throw "directory survived recursive delete" }
Write-Host "::endgroup::"
- name: Logs
if: always()
shell: pwsh
run: |
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-remount.log','C:\seaweed-remount.err.log','C:\seaweed-remount2.log','C:\seaweed-remount2.err.log','C:\seaweed-dirmount.log','C:\seaweed-dirmount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
}
-77
View File
@@ -1,77 +0,0 @@
name: "Multi-Master Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/multi-master-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
multi-master-failover-tests:
name: Multi-Master Failover Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Run multi-master failover tests
# The tests in test/multi_master spin up their own 3-node master raft
# cluster (using the freshly-installed `weed` binary) and exercise
# leader-election, failover and recovery scenarios. The shared
# test/testutil port-allocator regression test runs alongside since it
# is a prerequisite for the cluster fixtures.
run: |
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
- name: Collect server logs on failure
if: failure()
run: |
# test/multi_master/cluster.go creates per-test dirs via
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
# node's log into <baseDir>/logs/master*.log.
echo "Collecting per-node master logs from temp directories..."
mkdir -p /tmp/multi-master-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
echo "Found test directory: $dir"
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
done
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: multi-master-test-logs
path: /tmp/multi-master-logs/
retention-days: 7
-477
View File
@@ -1,477 +0,0 @@
name: "Performance"
on:
push:
branches: [ master ]
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- 'seaweed-volume/**'
- 'test/perf/**'
- '.github/workflows/performance.yml'
workflow_dispatch:
inputs:
profile_duration:
description: "CPU profiling duration in seconds"
required: false
default: "30"
type: string
benchmark_files:
description: "Number of files for the throughput benchmark"
required: false
default: "100000"
type: string
benchmark_concurrency:
description: "Concurrent read/write workers"
required: false
default: "16"
type: string
benchmark_size:
description: "Simulated file size in bytes"
required: false
default: "1024"
type: string
s3_objects:
description: "Number of objects for the S3 benchmark"
required: false
default: "20000"
type: string
s3_size:
description: "S3 object size in bytes"
required: false
default: "4096"
type: string
concurrency:
group: ${{ github.head_ref || github.ref }}/performance
cancel-in-progress: true
permissions:
contents: read
env:
VOL_SIZE_LIMIT: "1024"
jobs:
performance-profile:
name: CPU and Heap Profile
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build weed
run: go build -o weed_bin ./weed
- name: Start server with profiling enabled
run: |
mkdir -p ./perfdata
./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \
-s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \
-s3.port=8000 -s3.config=./docker/compose/s3.json \
> weed.log 2>&1 &
echo "WEED_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
break
fi
sleep 1
done
# give the volume server a moment to register with the master
sleep 3
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Capture profiles under load
run: |
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
# drive write load so the sampled profile reflects real work
./weed_bin benchmark -master=localhost:9333 -writeOnly \
-c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 &
echo "Sampling CPU profile for ${DURATION}s..."
curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof
curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof
curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS)"
echo '```'
if [ -f mem-profile.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Profile summary
if: always()
run: |
{
echo "## CPU profile (top functions)"
echo '```'
head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop server
if: always()
run: kill "${WEED_PID}" 2>/dev/null || true
- name: Show server log on failure
if: failure()
run: tail -200 weed.log || true
- name: Upload profiles
if: always()
uses: actions/upload-artifact@v7
with:
name: performance-profile-${{ github.run_number }}
path: |
cpu.pprof
heap.pprof
cpu-top.txt
heap-top.txt
goroutine.txt
mem-profile.csv
mem-profile.csv.peak
weed.log
retention-days: 30
benchmark:
name: Throughput Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
impl: [go, rust]
env:
IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
if: matrix.impl == 'rust'
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build weed
run: go build -o weed_bin ./weed
- name: Build Rust volume server
if: matrix.impl == 'rust'
run: cd seaweed-volume && cargo build --release
- name: Start master and volume server
run: |
mkdir -p ./perfdata/master ./perfdata/vol
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
-mdir=./perfdata/master -peers=none \
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
> master.log 2>&1 &
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
break
fi
sleep 1
done
if [ "${IMPL}" = "rust" ]; then
./seaweed-volume/target/release/weed-volume \
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
> volume.log 2>&1 &
else
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
-port=8080 -dir=./perfdata/vol -max=100 \
> volume.log 2>&1 &
fi
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
echo "volume server is ready"
break
fi
sleep 1
done
# let the volume server register with the master via heartbeat
sleep 3
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-benchmark.csv \
"master=${MASTER_PID}" "volume=${VOLUME_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Run throughput benchmark
run: |
N="${{ github.event.inputs.benchmark_files || '100000' }}"
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
SIZE="${{ github.event.inputs.benchmark_size || '1024' }}"
./weed_bin benchmark -master=localhost:9333 \
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
- name: Run Go micro-benchmarks
if: matrix.impl == 'go'
continue-on-error: true
run: |
go test -run='^$' -bench=. -benchmem -benchtime=10x \
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
2>&1 | tee go-benchmarks.txt
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS, ${IMPL} volume)"
echo '```'
if [ -f mem-benchmark.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Benchmark summary
if: always()
run: |
{
echo "## Throughput benchmark (${IMPL} volume)"
echo '```'
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop processes
if: always()
run: |
kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- name: Show logs on failure
if: failure()
run: |
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
- name: Upload benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
benchmark-results.txt
go-benchmarks.txt
mem-benchmark.csv
mem-benchmark.csv.peak
master.log
volume.log
retention-days: 7
s3-benchmark:
name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
impl: [go, rust]
env:
IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
if: matrix.impl == 'rust'
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build weed and S3 load tool
run: |
go build -o weed_bin ./weed
go build -o s3bench ./test/s3/benchmark
- name: Build Rust volume server
if: matrix.impl == 'rust'
run: cd seaweed-volume && cargo build --release
- name: Start cluster with S3 gateway
run: |
mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
-mdir=./perfdata/master -peers=none \
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
> master.log 2>&1 &
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
break
fi
sleep 1
done
if [ "${IMPL}" = "rust" ]; then
./seaweed-volume/target/release/weed-volume \
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
> volume.log 2>&1 &
else
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
-port=8080 -dir=./perfdata/vol -max=100 \
> volume.log 2>&1 &
fi
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
echo "volume server is ready"
break
fi
sleep 1
done
sleep 3
./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
-s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
> filer.log 2>&1 &
echo "FILER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 30); do
if nc -z localhost 8000 2>/dev/null; then
echo "s3 gateway is ready"
break
fi
sleep 1
done
sleep 2
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-s3.csv \
"master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Run S3 read/write benchmark
run: |
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
SIZE="${{ github.event.inputs.s3_size || '4096' }}"
./s3bench -endpoint=http://localhost:8000 \
-access-key=some_access_key1 -secret-key=some_secret_key1 \
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
2>&1 | tee s3-benchmark-results.txt
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS, ${IMPL} volume)"
echo '```'
if [ -f mem-s3.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: S3 benchmark summary
if: always()
run: |
{
echo "## S3 read/write benchmark (${IMPL} volume)"
echo '```'
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop processes
if: always()
run: |
kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- name: Show logs on failure
if: failure()
run: |
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
- name: Upload S3 benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
s3-benchmark-results.txt
mem-s3.csv
mem-s3.csv.peak
master.log
volume.log
filer.log
retention-days: 7
-118
View File
@@ -1,118 +0,0 @@
name: "pjdfstest POSIX Compliance"
on:
push:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
workflow_dispatch:
concurrency:
group: pjdfstest/${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
pjdfstest:
name: pjdfstest
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Start local Docker registry
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Build weed race binary
run: |
cd docker
make binary_race
- name: Build SeaweedFS e2e image
uses: docker/build-push-action@v7
with:
context: docker
file: docker/Dockerfile.e2e
tags: localhost:5000/chrislusf/seaweedfs:e2e
push: true
cache-from: type=gha,scope=pjdfstest-e2e
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
- name: Tag e2e image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:e2e
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
- name: Build pjdfstest image
uses: docker/build-push-action@v7
with:
context: test/pjdfstest
build-contexts: |
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
push: true
cache-from: type=gha,scope=pjdfstest-harness
cache-to: type=gha,mode=max,scope=pjdfstest-harness
- name: Tag pjdfstest image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
- name: Start SeaweedFS cluster
run: |
docker compose -f test/pjdfstest/docker-compose.yml up --wait
- name: Run pjdfstest
run: |
set -o pipefail
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/pjdfstest-docker-logs
for svc in master volume filer mount; do
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
done
- name: Tear down
if: always()
run: |
docker compose -f test/pjdfstest/docker-compose.yml down -v
- name: Upload logs
if: always()
uses: actions/upload-artifact@v7
with:
name: pjdfstest-results
path: |
/tmp/pjdfstest-output.log
/tmp/pjdfstest-docker-logs/
retention-days: 7
+2 -14
View File
@@ -3,20 +3,8 @@ name: "Plugin Worker Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/plugin_workers/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/plugin-workers.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/plugin_workers/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/plugin-workers.yml'
permissions:
contents: read
@@ -39,13 +27,13 @@ jobs:
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.26
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Run plugin worker tests
run: go test -v ./${{ matrix.path }}
+3 -24
View File
@@ -3,24 +3,8 @@ name: "PostgreSQL Gateway Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/postgres/**'
- 'weed/query/**'
- 'weed/mq/**'
- 'test/postgres/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/postgres-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/postgres/**'
- 'weed/query/**'
- 'weed/mq/**'
- 'test/postgres/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/postgres-tests.yml'
concurrency:
group: ${{ github.head_ref }}/postgres-tests
@@ -39,24 +23,19 @@ jobs:
working-directory: test/postgres
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v7
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
uses: actions/checkout@v6
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
-115
View File
@@ -1,115 +0,0 @@
name: "release: bump version"
on:
workflow_dispatch:
inputs:
bump:
description: "Which part to increment (ignored when 'version' is set)"
type: choice
options:
- minor
- major
default: minor
version:
description: "Explicit MAJOR.MINOR to set, e.g. 4.36 (overrides 'bump')"
type: string
required: false
permissions:
contents: write
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Compute new version
id: compute
env:
BUMP: ${{ inputs.bump }}
INPUT_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
CONST=weed/util/version/constants.go
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
echo "current: ${MAJOR}.${MINOR}"
if [ -n "$INPUT_VERSION" ]; then
if ! [[ "$INPUT_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
echo "::error::version must be MAJOR.MINOR, e.g. 5.01 (got '$INPUT_VERSION')"
exit 1
fi
# 10# forces base 10 so 08/09 are not parsed as octal.
MAJOR=$((10#${INPUT_VERSION%%.*}))
MINOR=$((10#${INPUT_VERSION##*.}))
if [ "$MINOR" -gt 99 ]; then
echo "::error::minor must be 0-99 (got $MINOR); it rolls into the next major at 99"
exit 1
fi
else
case "$BUMP" in
# Minor is a 2-digit field: 4.99 -> 5.00 -> 5.01.
major) MAJOR=$((MAJOR + 1)); MINOR=0 ;;
minor)
if [ "$MINOR" -ge 99 ]; then
MAJOR=$((MAJOR + 1)); MINOR=0
else
MINOR=$((MINOR + 1))
fi
;;
*) echo "::error::unknown bump '$BUMP'"; exit 1 ;;
esac
fi
# appVersion mirrors the Go VERSION_NUMBER (zero-padded minor);
# chart version is plain SemVer (no leading zeros).
APP_VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
CHART_VERSION="${MAJOR}.${MINOR}.0"
echo "new: app=${APP_VERSION} chart=${CHART_VERSION}"
{
echo "major=${MAJOR}"
echo "minor=${MINOR}"
echo "app_version=${APP_VERSION}"
echo "chart_version=${CHART_VERSION}"
} >> "$GITHUB_OUTPUT"
- name: Apply version to constants.go
env:
MAJOR: ${{ steps.compute.outputs.major }}
MINOR: ${{ steps.compute.outputs.minor }}
run: |
set -euo pipefail
CONST=weed/util/version/constants.go
sed -i -E "s/(MAJOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MAJOR}\2/" "$CONST"
sed -i -E "s/(MINOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MINOR}\2/" "$CONST"
grep -E 'MAJOR_VERSION|MINOR_VERSION' "$CONST"
- name: Apply version to Chart.yaml
env:
APP_VERSION: ${{ steps.compute.outputs.app_version }}
CHART_VERSION: ${{ steps.compute.outputs.chart_version }}
run: |
set -euo pipefail
CHART=k8s/charts/seaweedfs/Chart.yaml
sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "$CHART"
sed -i -E "s/^version:.*/version: ${CHART_VERSION}/" "$CHART"
cat "$CHART"
- name: Commit and push
env:
APP_VERSION: ${{ steps.compute.outputs.app_version }}
run: |
set -euo pipefail
if git diff --quiet; then
echo "No version change to commit."
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add weed/util/version/constants.go k8s/charts/seaweedfs/Chart.yaml
git commit -m "${APP_VERSION}"
git push
+52 -54
View File
@@ -34,7 +34,7 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
@@ -42,22 +42,16 @@ jobs:
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
rust-
- name: Build Rust volume server
run: cd seaweed-volume && cargo build --release
@@ -72,10 +66,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -85,27 +79,21 @@ jobs:
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
rust-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
go build -o weed .
chmod +x weed
./weed version
@@ -146,9 +134,6 @@ jobs:
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
env:
# Keep in step with the length of matrix.shard below.
SHARD_COUNT: 3
strategy:
fail-fast: false
matrix:
@@ -157,10 +142,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -170,55 +155,51 @@ jobs:
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
rust-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
go build -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
# the 114 grpc tests and ran nearly twice as long as the other two shards.
- name: Select this shard's tests
env:
TEST_TYPE: ${{ matrix.test-type }}
SHARD: ${{ matrix.shard }}
run: |
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
# An empty list would make -run match nothing and the shard pass vacuously.
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
- name: Run volume server integration tests with Rust volume
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
VOLUME_SERVER_IMPL: rust
run: |
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
- name: Collect logs on failure
@@ -239,6 +220,23 @@ jobs:
- name: Test summary
if: always()
run: |
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
+6 -7
View File
@@ -19,7 +19,6 @@ jobs:
steps:
- name: Delete old Rust volume dev assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
@@ -39,7 +38,7 @@ jobs:
asset_suffix: linux-amd64
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
@@ -48,7 +47,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
@@ -84,7 +83,7 @@ jobs:
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
tag_name: dev
prerelease: true
@@ -108,7 +107,7 @@ jobs:
asset_suffix: darwin-amd64
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: brew install protobuf
@@ -119,7 +118,7 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
@@ -155,7 +154,7 @@ jobs:
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
tag_name: dev
prerelease: true
+22 -91
View File
@@ -26,7 +26,7 @@ jobs:
cross: true
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
@@ -39,19 +39,11 @@ jobs:
- name: Install cross-compilation tools
if: matrix.cross
run: |
sudo dpkg --add-architecture arm64
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
@@ -66,56 +58,36 @@ jobs:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
cargo build --release --target ${{ matrix.target }}
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
cargo build --release --target ${{ matrix.target }} --no-default-features
- name: Package binaries
run: |
# Large disk (default, 5bytes feature)
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
rm weed-volume-large-disk
# Normal volume size
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
rm weed-volume-normal
- name: Generate md5 checksums
run: |
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
md5sum "$f" > "$f.md5"
done
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.asset_suffix }}
path: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
build-rust-volume-darwin:
permissions:
contents: write
@@ -129,7 +101,7 @@ jobs:
asset_suffix: darwin_arm64
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: brew install protobuf
@@ -140,7 +112,7 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
@@ -155,61 +127,41 @@ jobs:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
cargo build --release --target ${{ matrix.target }}
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
cargo build --release --target ${{ matrix.target }} --no-default-features
- name: Package binaries
run: |
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
rm weed-volume-large-disk
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
rm weed-volume-normal
- name: Generate md5 checksums
run: |
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
md5 -r "$f" > "$f.md5"
done
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.asset_suffix }}
path: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
build-rust-volume-windows:
permissions:
contents: write
runs-on: windows-latest
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: choco install protoc -y
@@ -218,7 +170,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
@@ -233,52 +185,31 @@ jobs:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target-dir target/large-disk
cargo build --release
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --no-default-features --target-dir target/normal
cargo build --release --no-default-features
- name: Package binaries
shell: bash
run: |
cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe
cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
rm weed-volume-large-disk.exe
cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe
cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
rm weed-volume-normal.exe
- name: Generate md5 checksums
shell: bash
run: |
for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do
md5sum "$f" > "$f.md5"
done
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
files: |
weed-volume_large_disk_windows_amd64.zip
weed-volume_large_disk_windows_amd64.zip.md5
weed-volume_windows_amd64.zip
weed-volume_windows_amd64.zip.md5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-windows_amd64
path: |
weed-volume_large_disk_windows_amd64.zip
weed-volume_large_disk_windows_amd64.zip.md5
weed-volume_windows_amd64.zip
weed-volume_windows_amd64.zip.md5
-129
View File
@@ -1,129 +0,0 @@
name: "S3 ETag and ACL Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-etag-acl-tests:
name: S3 ETag + ACL Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Start weed mini (S3 on :8333)
run: |
mkdir -p /tmp/seaweedfs-etag-acl
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
{
"identities": [
{
"name": "admin",
"credentials": [
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
],
"actions": ["Admin", "Read", "Write"]
}
]
}
JSON
AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
weed mini \
-dir=/tmp/seaweedfs-etag-acl \
-s3.port=8333 \
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
-ip=127.0.0.1 \
> /tmp/weed-mini.log 2>&1 &
echo $! > /tmp/weed-mini.pid
# Wait for the S3 endpoint to come up (returns 403 unauth before any
# request is signed; that's fine — it means the server is listening).
for i in $(seq 1 30); do
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
echo "weed mini is ready"
exit 0
fi
sleep 1
done
echo "weed mini failed to start within 30s"
tail -50 /tmp/weed-mini.log
exit 1
- name: Run ETag tests
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
# SDK for Java v2 rejects the latter on the PutObject path.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/etag/...
- name: Run ACL versioning tests
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
# PutObjectAcl with and without versionId, modifying ACLs on different
# versions independently.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/acl/...
- name: Stop weed mini
if: always()
run: |
if [ -f /tmp/weed-mini.pid ]; then
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
fi
- name: Show server log on failure
if: failure()
run: |
echo "=== weed mini log (last 200 lines) ==="
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
- name: Archive log
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-etag-acl-server-log
path: /tmp/weed-mini.log
retention-days: 3
@@ -2,16 +2,7 @@ name: "S3 Authenticated Integration Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/iam/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/normal/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-example-integration-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-integration-tests
cancel-in-progress: true
@@ -27,10 +18,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+3 -11
View File
@@ -2,15 +2,7 @@ name: "S3 Filer Group Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/filer_group/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-filer-group-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-filer-group-tests
cancel-in-progress: true
@@ -30,10 +22,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+19 -153
View File
@@ -2,15 +2,7 @@ name: "S3 Go Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-go-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-go-tests
cancel-in-progress: true
@@ -33,10 +25,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -97,10 +89,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -145,10 +137,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -196,10 +188,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -253,132 +245,6 @@ jobs:
path: test/s3/retention/weed-test*.log
retention-days: 3
s3-lifecycle-tests:
name: S3 Lifecycle Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
# One job per test so each gets a fresh `weed mini` server, avoiding
# the cross-test volume-pool exhaustion that surfaced when several
# TTL-pinned bucket collections piled up in a single run.
test:
- TestLifecycleAbortIncompleteMultipartUpload
- TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
- TestLifecycleBootstrapWalkOnExistingObjects
- TestLifecycleConfigUpdateBetweenSweeps
- TestLifecycleDeleteBucketLifecycleStopsDispatching
- TestLifecycleDisabledRuleSkipsObject
- TestLifecycleEmptyBucketSweepIsNoOp
- TestLifecycleExpirationDateInThePast
- TestLifecycleExpirationFiresOnBackdatedObject
- TestLifecycleExpiredDeleteMarkerCleanup
- TestLifecycleMultipleBucketsInOneSweep
- TestLifecycleMultipleRulesInOneBucket
- TestLifecycleNewerNoncurrentVersions
- TestLifecycleNoncurrentVersionExpiration
- TestLifecycleSizeFilterGreaterThan
- TestLifecycleSkipsObjectLockedObjects
- TestLifecycleSuspendedVersioningExpiration
- TestLifecycleTagFilter
- TestLifecycleVersionedBucketCreatesDeleteMarker
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run ${{ matrix.test }}
timeout-minutes: 8
working-directory: test/s3/lifecycle
run: |
set -x
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
- name: Show server logs on failure
if: failure()
working-directory: test/s3/lifecycle
run: |
if [ -f weed-test.log ]; then
echo "=== Last 200 lines of server logs ==="
tail -200 weed-test.log
fi
ps aux | grep -E "(weed|test)" || true
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-lifecycle-test-logs-${{ matrix.test }}
path: test/s3/lifecycle/weed-test*.log
retention-days: 3
s3-checksum-tests:
name: S3 Checksum Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Checksum Tests
timeout-minutes: 16
working-directory: test/s3/checksum
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Tests ==="
make test-with-server
- name: Show server logs on failure
if: failure()
working-directory: test/s3/checksum
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-checksum-test-logs
path: test/s3/checksum/weed-test*.log
retention-days: 3
s3-cors-tests:
name: S3 CORS Tests
runs-on: ubuntu-22.04
@@ -389,10 +255,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -453,10 +319,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -502,10 +368,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -570,10 +436,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -625,10 +491,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+9 -26
View File
@@ -39,10 +39,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -84,14 +84,14 @@ jobs:
timeout-minutes: 25
strategy:
matrix:
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
test-type: ["basic", "advanced", "policy-enforcement", "group"]
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -138,23 +138,6 @@ jobs:
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestIAMGroup" ./...
;;
"sts")
echo "Running STS and service account tests..."
make clean setup start-services wait-for-services
# SigV4-signed STS calls need admin credentials matching test_config.json.
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
export STS_TEST_ACCESS_KEY=test-access-key
export STS_TEST_SECRET_KEY=test-secret-key
# The use_service_account_credentials subtest is excluded because
# newly-created service-account access keys are not currently
# persisted to the filer after CreateServiceAccount — a
# pre-existing sync issue tracked separately from the
# GetFederationToken routing fix this PR addresses.
go test -v -timeout 15m \
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
./...
;;
*)
echo "Unknown test type: ${{ matrix.test-type }}"
exit 1
@@ -202,10 +185,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -262,10 +245,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+2 -2
View File
@@ -35,10 +35,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -26,10 +26,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -38,22 +38,7 @@ jobs:
working-directory: test/s3/versioning
run: |
set -x
# Run every versioning test, so a regression test lands covered instead
# of waiting for someone to remember this file. Name a test in EXCLUDE,
# with the reason, to keep it out.
#
# TestVersioningPagination*: opt-in stress tests that build 1500+
# versions. They self-skip without ENABLE_STRESS_TESTS and have their
# own make target, so this gate should not carry them.
EXCLUDE='TestVersioningPagination.*'
tests=$(go test . -list '.*' | grep '^Test' | sort -u)
# An empty list would make -run match nothing and pass this job vacuously.
[ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; }
selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$")
[ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; }
echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests"
# make swallows a lone trailing $, taking the anchor with it, so escape it.
make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$'
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
- name: Show server logs on failure
if: failure()
@@ -79,10 +64,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -117,10 +102,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
+5 -5
View File
@@ -36,16 +36,16 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
@@ -143,10 +143,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
+10 -10
View File
@@ -43,10 +43,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -86,10 +86,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -195,10 +195,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -313,10 +313,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -400,10 +400,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+4 -25
View File
@@ -3,22 +3,8 @@ name: "S3 Proxy Signature Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'test/s3/proxy_signature/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-proxy-signature-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'test/s3/proxy_signature/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-proxy-signature-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
@@ -34,19 +20,14 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
@@ -90,10 +71,8 @@ jobs:
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
S3_READY=0
for i in $(seq 1 30); do
# Check logs first for the readiness line. weed mini's progress
# board prints " S3 ready (Xs)"; older builds and the
# standalone S3 binary log "S3 (gateway|service) ... ready".
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
# Check logs first for startup message (weed mini says "S3 service is ready")
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)"; then
echo "SeaweedFS S3 gateway is ready"
S3_READY=1
break
@@ -1,110 +0,0 @@
name: "S3 SDK V2 Route Disambiguation Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'test/s3/sdk_v2_routing/**'
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'test/s3/sdk_v2_routing/**'
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-sdk-v2-routing-tests:
name: S3 SDK V2 Routing Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Start weed mini (S3 on :8333)
# Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a
# listing a bucket literally named "buckets" must get an XML
# ListObjectsV2 response, not the JSON ListTableBuckets body
# served by the S3 Tables REST endpoint on the same path.
run: |
mkdir -p /tmp/seaweedfs-sdk-v2-routing
cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON'
{
"identities": [
{
"name": "admin",
"credentials": [
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
],
"actions": ["Admin", "Read", "Write"]
}
]
}
JSON
AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
weed mini \
-dir=/tmp/seaweedfs-sdk-v2-routing \
-s3.port=8333 \
-s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \
-ip=127.0.0.1 \
> /tmp/weed-mini.log 2>&1 &
echo $! > /tmp/weed-mini.pid
for i in $(seq 1 30); do
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
echo "weed mini is ready"
exit 0
fi
sleep 1
done
echo "weed mini failed to start within 30s"
tail -50 /tmp/weed-mini.log
exit 1
- name: Run SDK V2 routing tests
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/...
- name: Stop weed mini
if: always()
run: |
if [ -f /tmp/weed-mini.pid ]; then
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
fi
- name: Show server log on failure
if: failure()
run: |
echo "=== weed mini log (last 200 lines) ==="
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
- name: Archive log
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-sdk-v2-routing-server-log
path: /tmp/weed-mini.log
retention-days: 3
+5 -9
View File
@@ -25,27 +25,23 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Pre-pull Spark image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/spark:3.5.1
run: docker pull apache/spark:3.5.8
- name: Run S3 Spark integration tests
working-directory: test/s3/spark
+16 -20
View File
@@ -44,10 +44,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -73,12 +73,8 @@ jobs:
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
else
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
# same change that fixed them.
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
fi
- name: Show server logs on failure
@@ -112,10 +108,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -160,10 +156,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -209,10 +205,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -258,10 +254,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -309,10 +305,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -357,10 +353,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+50 -315
View File
@@ -2,14 +2,10 @@ name: "S3 Tables Integration Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3tables/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-tables-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-tables-tests
cancel-in-progress: true
permissions:
contents: read
@@ -23,10 +19,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -83,19 +79,14 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Run go mod tidy
run: go mod tidy
@@ -144,23 +135,19 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Trino image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull trinodb/trino:479
run: docker pull trinodb/trino:479
- name: Run go mod tidy
run: go mod tidy
@@ -207,152 +194,6 @@ jobs:
path: test/s3tables/catalog_trino/test-output.log
retention-days: 3
dremio-iceberg-catalog-tests:
name: Dremio Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull images
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull dremio/dremio-oss:25.2.0
pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Dremio Iceberg Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog_dremio
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Dremio Iceberg Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Dremio Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_dremio
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|dremio)" || true
echo "=== Dremio containers ==="
docker ps -a --filter "name=seaweed-dremio" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: dremio-iceberg-catalog-test-logs
path: test/s3tables/catalog_dremio/test-output.log
retention-days: 3
doris-iceberg-catalog-tests:
name: Doris Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 35
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull images
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/doris:doris-all-in-one-2.1.0
pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Doris Iceberg Catalog Integration Tests
timeout-minutes: 30
working-directory: test/s3tables/catalog_doris
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Doris Iceberg Catalog Tests ==="
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
echo "Doris Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_doris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|doris)" || true
echo "=== Doris containers ==="
docker ps -a --filter "name=seaweed-doris" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: doris-iceberg-catalog-test-logs
path: test/s3tables/catalog_doris/test-output.log
retention-days: 3
polaris-integration-tests:
name: Polaris Integration Tests
runs-on: ubuntu-22.04
@@ -360,10 +201,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -375,15 +216,8 @@ jobs:
run: |
go install -buildvcs=false ./weed
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull Polaris image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/polaris:latest
run: docker pull apache/polaris:latest
- name: Run Polaris Integration Tests
timeout-minutes: 25
@@ -428,23 +262,19 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Spark image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/spark:3.5.1
run: docker pull apache/spark:3.5.1
- name: Run go mod tidy
run: go mod tidy
@@ -498,24 +328,21 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull RisingWave image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull risingwavelabs/risingwave:v2.5.0
pull postgres:16-alpine
docker pull risingwavelabs/risingwave:v2.5.0
docker pull postgres:16-alpine
- name: Run go mod tidy
run: go mod tidy
@@ -569,47 +396,19 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Week stamp for image cache key
id: week
run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
- name: Restore python:3 image cache
id: python-image
uses: actions/cache@v6
with:
path: /tmp/python3-image.tar
key: python3-image-${{ steps.week.outputs.week }}
restore-keys: |
python3-image-
- name: Load or pull python:3
run: |
if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then
docker load -i /tmp/python3-image.tar
exit 0
fi
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
if pull python:3; then
docker save -o /tmp/python3-image.tar python:3
elif [ -f /tmp/python3-image.tar ]; then
# Docker Hub unreachable; fall back to last week's cached image
docker load -i /tmp/python3-image.tar
else
exit 1
fi
- name: Pre-pull Python image
run: docker pull python:3
- name: Run go mod tidy
run: go mod tidy
@@ -663,14 +462,23 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Python image
run: docker pull python:3
- name: Pre-pull LocalStack image (if needed)
run: docker pull localstack/localstack:latest || true
- name: Run go mod tidy
run: go mod tidy
@@ -716,79 +524,6 @@ jobs:
path: test/s3tables/lakekeeper/test-output.log
retention-days: 3
unity-catalog-integration-tests:
name: Unity Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull images
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull unitycatalog/unitycatalog:v0.4.0
pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Unity Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/unity_catalog
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Unity Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Unity Catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/unity_catalog
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
echo "=== Unity Catalog containers ==="
docker ps -a --filter "name=seaweed-unity-catalog" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: unity-catalog-integration-test-logs
path: test/s3tables/unity_catalog/test-output.log
retention-days: 3
s3-tables-build-verification:
name: S3 Tables Build Verification
runs-on: ubuntu-22.04
@@ -796,10 +531,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -861,10 +596,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -912,10 +647,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+29 -126
View File
@@ -3,26 +3,8 @@ name: "Ceph S3 tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'weed/iam/**'
- 'test/s3/compatibility/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'weed/iam/**'
- 'test/s3/compatibility/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3tests
@@ -38,16 +20,16 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -55,15 +37,13 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
python3 test/s3/fix_s3_tests_bucket_conflicts.py
env:
S3_TESTS_PATH: s3-tests
@@ -150,49 +130,7 @@ jobs:
done
echo "✅ S3 server is responding, starting tests..."
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
# something driving deletions. The s3tests build tag rescales one
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
# the upload's mtime; -dispatch / -checkpoint defaults are already
# tightened under the same build tag.
LC_LOG=/tmp/lifecycle-worker.log
# -debug routes glog to stderr so the bootstrap walker's progress
# shows up in $LC_LOG; without it weed shell silences glog.
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
| weed shell -debug -master=localhost:9333 \
> "$LC_LOG" 2>&1 &
lc_pid=$!
# Aliveness check: a bad shell command exits in <1s and the suite
# would otherwise just timeout the expiration tests with no signal.
sleep 2
if ! kill -0 "$lc_pid" 2>/dev/null; then
echo "lifecycle worker died on startup"
tail -50 "$LC_LOG" 2>/dev/null || true
exit 1
fi
echo "lifecycle worker pid=$lc_pid"
# bash -e exits the step on the first tox failure, so move teardown
# into a trap to guarantee the worker log + data dir reach the runner.
cleanup() {
status=$?
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
# bash fallback if it ignores TERM. Reading the log AFTER the
# graceful-stop window catches the bootstrap walker's progress.
kill -TERM "$lc_pid" 2>/dev/null || true
kill -TERM "$pid" 2>/dev/null || true
sleep 1
if [ "$status" -ne 0 ]; then
echo "=== lifecycle worker log (tail) ==="
tail -200 "$LC_LOG" 2>/dev/null || true
fi
kill -9 "$lc_pid" 2>/dev/null || true
kill -9 "$pid" 2>/dev/null || true
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
}
trap cleanup EXIT
tox -- \
s3tests/functional/test_s3.py::test_bucket_list_empty \
s3tests/functional/test_s3.py::test_bucket_list_distinct \
@@ -372,8 +310,11 @@ jobs:
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
# cleanup() trap handles worker/server kill + data dir wipe.
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
versioning-tests:
name: S3 Versioning & Object Lock tests
@@ -381,16 +322,16 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -398,15 +339,13 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
python3 test/s3/fix_s3_tests_bucket_conflicts.py
env:
S3_TESTS_PATH: s3-tests
@@ -551,16 +490,16 @@ jobs:
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -568,8 +507,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -676,10 +613,10 @@ jobs:
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -694,12 +631,10 @@ jobs:
export WEED_DATA_DIR="/tmp/seaweedfs-copy-test-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
set -x
# Each test bucket is its own collection and grows 7 volumes; the suite runs
# faster than the heartbeat that returns slots from the deleted collections.
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=300 -volume.preStopSeconds=1 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
pid=$!
@@ -767,7 +702,7 @@ jobs:
sleep 2
done
MASTER_ENDPOINT="http://127.0.0.1:9336" go test -v
go test -v
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
@@ -778,16 +713,16 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -795,8 +730,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -1017,39 +950,6 @@ jobs:
sleep 2
done
# Spawn the lifecycle worker (see basic-tests block for context).
LC_LOG=/tmp/lifecycle-worker-sql.log
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
| weed shell -debug -master=localhost:9337 \
> "$LC_LOG" 2>&1 &
lc_pid=$!
sleep 2
if ! kill -0 "$lc_pid" 2>/dev/null; then
echo "lifecycle worker died on startup"
tail -50 "$LC_LOG" 2>/dev/null || true
exit 1
fi
echo "lifecycle worker pid=$lc_pid"
cleanup() {
status=$?
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
# bash fallback if it ignores TERM. Reading the log AFTER the
# graceful-stop window catches the bootstrap walker's progress.
kill -TERM "$lc_pid" 2>/dev/null || true
kill -TERM "$pid" 2>/dev/null || true
sleep 1
if [ "$status" -ne 0 ]; then
echo "=== lifecycle worker log (tail) ==="
tail -200 "$LC_LOG" 2>/dev/null || true
fi
kill -9 "$lc_pid" 2>/dev/null || true
kill -9 "$pid" 2>/dev/null || true
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
}
trap cleanup EXIT
tox -- \
s3tests/functional/test_s3.py::test_bucket_list_empty \
s3tests/functional/test_s3.py::test_bucket_list_distinct \
@@ -1229,7 +1129,10 @@ jobs:
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
# cleanup() trap handles worker/server kill + data dir wipe.
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
-120
View File
@@ -1,120 +0,0 @@
name: "Samba on FUSE Integration"
on:
push:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'weed/cluster/**'
- 'test/samba/**'
- '.github/workflows/samba-integration.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'weed/cluster/**'
- 'test/samba/**'
- '.github/workflows/samba-integration.yml'
workflow_dispatch:
concurrency:
group: samba-integration/${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
samba-integration:
name: samba-integration
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Start local Docker registry
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Build weed race binary
run: |
cd docker
make binary_race
- name: Build SeaweedFS e2e image
uses: docker/build-push-action@v7
with:
context: docker
file: docker/Dockerfile.e2e
tags: localhost:5000/chrislusf/seaweedfs:e2e
push: true
cache-from: type=gha,scope=samba-e2e
cache-to: type=gha,mode=max,scope=samba-e2e
- name: Tag e2e image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:e2e
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
- name: Build samba image
uses: docker/build-push-action@v7
with:
context: test/samba
build-contexts: |
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
tags: localhost:5000/chrislusf/seaweedfs:samba
push: true
cache-from: type=gha,scope=samba-harness
cache-to: type=gha,mode=max,scope=samba-harness
- name: Tag samba image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:samba
docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba
- name: Start SeaweedFS cluster and Samba
run: |
docker compose -f test/samba/docker-compose.yml up --wait
- name: Run Samba test battery
run: |
set -o pipefail
docker compose -f test/samba/docker-compose.yml exec -T samba \
/run_inside_container.sh 2>&1 | tee /tmp/samba-output.log
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/samba-docker-logs
for svc in master volume filer samba; do
docker compose -f test/samba/docker-compose.yml logs "$svc" \
> "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true
done
- name: Tear down
if: always()
run: |
docker compose -f test/samba/docker-compose.yml down -v
- name: Upload logs
if: always()
uses: actions/upload-artifact@v7
with:
name: samba-integration-results
path: |
/tmp/samba-output.log
/tmp/samba-docker-logs/
retention-days: 7
+2 -2
View File
@@ -34,10 +34,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -31,7 +31,7 @@ jobs:
# SETUP & BUILD
# ========================================
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up JDK 11
uses: actions/setup-java@v5
@@ -41,7 +41,7 @@ jobs:
cache: maven
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -150,7 +150,7 @@ jobs:
- name: Cache Apache Spark
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
id: cache-spark
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: spark-3.5.0-bin-hadoop3
key: spark-3.5.0-hadoop3
+2 -2
View File
@@ -24,10 +24,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
-80
View File
@@ -1,80 +0,0 @@
name: "terraform: validate and test modules"
on:
push:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
pull_request:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
permissions:
contents: read
jobs:
validate:
name: fmt, validate, plan-level tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up OpenTofu
uses: opentofu/setup-opentofu@v2
with:
tofu_version: 1.12.1
- name: fmt check
working-directory: terraform
run: tofu fmt -recursive -check -diff
- name: validate core
working-directory: terraform/modules/core
run: |
tofu init -backend=false -input=false
tofu validate
- name: validate security
working-directory: terraform/modules/security
run: |
tofu init -backend=false -input=false
tofu validate
- name: plan-level tests (core)
working-directory: terraform/modules/core
run: tofu test
- name: validate examples
run: |
set -e
for ex in terraform/examples/*/; do
echo "== validate $ex =="
tofu -chdir="$ex" init -backend=false -input=false
tofu -chdir="$ex" validate
done
smoke:
name: local cluster smoke test (real weed)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Build weed
run: go build -o "$RUNNER_TEMP/weed" ./weed
- name: Set up OpenTofu
uses: opentofu/setup-opentofu@v2
with:
tofu_version: 1.12.1
- name: Run local cluster harness
working-directory: terraform/test/local
run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
- name: Run local mTLS cluster harness
working-directory: terraform/test/local-secure
run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh
@@ -3,20 +3,8 @@ name: "test s3 over https using aws-cli"
on:
push:
branches: [master, test-https-s3-awscli]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/test-s3-over-https-using-awscli.yml'
pull_request:
branches: [master, test-https-s3-awscli]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/test-s3-over-https-using-awscli.yml'
env:
AWS_ACCESS_KEY_ID: some_access_key1
@@ -32,9 +20,9 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- uses: actions/setup-go@v7
- uses: actions/setup-go@v6
with:
go-version: ^1.25
-68
View File
@@ -1,68 +0,0 @@
name: "TLS Rotation Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/tls_rotation/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/tls-rotation-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/tls_rotation/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/tls-rotation-tests.yml'
permissions:
contents: read
jobs:
tls-rotation-tests:
name: TLS Rotation Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run TLS Rotation Integration Tests
working-directory: test/tls_rotation
run: |
go test -v -count=1 -timeout 5m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting master logs from temp directories..."
mkdir -p /tmp/tls-rotation-test-logs
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: tls-rotation-test-logs
path: /tmp/tls-rotation-test-logs/
retention-days: 14
+2 -9
View File
@@ -2,13 +2,6 @@ name: "TUS Protocol Tests"
on:
pull_request:
paths:
- 'weed/server/**'
- 'weed/filer/**'
- 'test/tus/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/tus-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/tus-tests
@@ -29,10 +22,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -1,68 +0,0 @@
name: "Vacuum Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/vacuum/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/vacuum-integration-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/vacuum/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/vacuum-integration-tests.yml'
permissions:
contents: read
jobs:
vacuum-integration-tests:
name: Vacuum Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run Vacuum Integration Tests
working-directory: test/vacuum
run: |
go test -v -timeout 10m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting server logs from temp directories..."
mkdir -p /tmp/vacuum-test-logs
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: vacuum-integration-test-logs
path: /tmp/vacuum-test-logs/
retention-days: 14
@@ -29,8 +29,6 @@ permissions:
env:
TEST_TIMEOUT: '30m'
# Keep in step with the length of matrix.shard below.
SHARD_COUNT: 3
jobs:
volume-server-integration-tests:
@@ -45,10 +43,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
@@ -59,26 +57,28 @@ jobs:
chmod +x weed
./weed version
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
# the 114 grpc tests and ran nearly twice as long as the other two shards.
- name: Select this shard's tests
env:
TEST_TYPE: ${{ matrix.test-type }}
SHARD: ${{ matrix.shard }}
run: |
tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
# An empty list would make -run match nothing and the shard pass vacuously.
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
- name: Run volume server integration tests
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: |
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
- name: Collect logs on failure
@@ -99,6 +99,23 @@ jobs:
- name: Test summary
if: always()
run: |
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
-1
View File
@@ -2,7 +2,6 @@
vendor
tags
*.swp
.claude/
### OSX template
.DS_Store
.AppleDouble
-5
View File
@@ -1,5 +0,0 @@
{
"setup": [],
"teardown": [],
"run": []
}
+2 -1
View File
@@ -42,7 +42,8 @@ test: admin-generate
# Admin component targets
admin-generate:
@cd $(ADMIN_DIR) && $(MAKE) generate
@echo "Generating admin component templates..."
@cd $(ADMIN_DIR) && templ generate ./view
admin-build: admin-generate
@echo "Building admin component..."
+27 -52
View File
@@ -35,7 +35,6 @@ Your support will be really appreciated by me and other supporters!
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
[![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca)
[![zyner](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_zyner.png)](https://zyner.org)
---
@@ -57,6 +56,7 @@ Table of Contents
* [Quick Start](#quick-start)
* [Quick Start with weed mini](#quick-start-with-weed-mini)
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
* [Quick Start with Single Binary](#quick-start-with-single-binary)
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
@@ -80,41 +80,39 @@ Table of Contents
## Quick Start with weed mini ##
The easiest way to get started with SeaweedFS for development and testing:
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file, or run `go install github.com/seaweedfs/seaweedfs/weed@latest`. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
Example:
```bash
AWS_ACCESS_KEY_ID=admin \
AWS_SECRET_ACCESS_KEY=secret \
S3_BUCKET=my-bucket \
# remove quarantine on macOS
# xattr -d com.apple.quarantine ./weed
./weed mini -dir=/data
```
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
The same command starts everything else too:
- **S3 Endpoint**: http://localhost:8333
This single command starts a complete SeaweedFS setup with:
- **Master UI**: http://localhost:9333
- **Volume Server**: http://localhost:9340
- **Filer UI**: http://localhost:8888
- **S3 Endpoint**: http://localhost:8333
- **WebDAV**: http://localhost:7333
- **Admin UI**: http://localhost:23646
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
Perfect for development, testing, learning SeaweedFS, and single-node deployments. To scale out, add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, on another machine, or on thousands of machines.
Perfect for development, testing, learning SeaweedFS, and single node deployments!
## Quick Start for S3 API on Docker ##
```bash
docker run -p 8333:8333 \
-e AWS_ACCESS_KEY_ID=admin \
-e AWS_SECRET_ACCESS_KEY=secret \
-e S3_BUCKET=my-bucket \
chrislusf/seaweedfs
```
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
Same behavior as the `weed mini` command above — the S3 endpoint is at http://localhost:8333 with `my-bucket` pre-created. Drop the env vars to run anonymously for development.
## Quick Start with Single Binary ##
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
# Introduction #
@@ -147,11 +145,6 @@ SeaweedFS can achieve both fast local access time and elastic cloud storage capa
What's more, the cloud storage access API cost is minimized.
Faster and cheaper than direct cloud storage!
SeaweedFS also ships a built-in **Iceberg REST Catalog**, turning the same cluster into a self-contained lakehouse.
Spark, Trino, Dremio, DuckDB, and RisingWave can query Iceberg tables directly — no Hive Metastore, Glue, or
external catalog service required. Storage and table metadata live in one system, simplifying on-prem and
small-team analytics stacks.
[Back to TOC](#table-of-contents)
# Features #
@@ -188,13 +181,6 @@ small-team analytics stacks.
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
## Data Lakehouse Features ##
* [S3 Table Buckets][S3TableBucket] expose a dedicated namespace for Iceberg tables with strict layout validation.
* Built-in [Iceberg REST Catalog][IcebergCatalog] runs alongside the S3 endpoint — no external metastore needed.
* Native integrations with [Apache Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], and [RisingWave][RisingWaveIceberg].
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan removal, manifest rewriting.
* Granular IAM at the bucket, namespace, and table level via standard S3 bucket policies.
## Kubernetes ##
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
@@ -218,14 +204,6 @@ small-team analytics stacks.
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket
[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration
[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration
[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration
[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
[Back to TOC](#table-of-contents)
@@ -426,7 +404,7 @@ If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage
## SeaweedFS Filer ##
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory structure is an interface that is implemented in many key-value stores or databases.
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory sturcture is an interface that is implemented in many key-value stores or databases.
The content of a file is mapped to one or many blobs, distributed to multiple volumes on multiple volume servers.
@@ -510,22 +488,20 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mong
### Compared to MinIO ###
Please note, as Apr 25, 2026 MinIO ceased development. It's strongly discouraged to use that unmaintained software with multiple security bugs.
MinIO follows AWS S3 closely and is ideal for testing for S3 API. It has good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. It is also possible to put MinIO as a gateway in front of SeaweedFS later.
MinIO followed AWS S3 closely and was ideal for testing for S3 API. It had good UI, policies, versionings, etc. SeaweedFS is trying to catch up here.
MinIO metadata are in simple files. Each file write will incur extra writes to corresponding meta file.
MinIO metadata were in simple files. Each file write will incur extra writes to corresponding meta file.
MinIO did not have optimization for lots of small files. The files were simply stored as is to local disks.
MinIO does not have optimization for lots of small files. The files are simply stored as is to local disks.
Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem.
MinIO had multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
MinIO has multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
MinIO had full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
MinIO has full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
MinIO did not have POSIX-like API support.
MinIO does not have POSIX-like API support.
MinIO had specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
MinIO has specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
## Dev Plan ##
@@ -655,8 +631,7 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
## Enterprise ##
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has advanced features, including data recovery, self-healing storage,
customizable erasure coding, EC vacuum and repair, etc.
which has a self-healing storage format with better data protection.
[Back to TOC](#table-of-contents)
File diff suppressed because it is too large Load Diff
-78
View File
@@ -1,78 +0,0 @@
# Security Policy
## Supported versions
Security fixes land in the latest release. Please reproduce against a recent
release or `master` before reporting; issues that only reproduce on old,
unsupported versions are not eligible for a fix or an advisory.
## Reporting a vulnerability
Report privately through GitHub private vulnerability reporting (the "Report a
vulnerability" button under the repository's Security tab). This keeps the
report, the fix, and any CVE in one place. Do not open a public issue.
### What a report must include
We can only act on reports that show real impact. Please include:
- Affected version (a release tag or `master` commit you reproduced on)
- The exact deployment and configuration: which components are running
(master, volume, filer, S3, admin), which ports are reachable by the
attacker, and what authentication is enabled
- The attacker's starting position: unauthenticated, a valid S3 user, an admin,
or someone with access to the internal cluster network
- The trust boundary that is crossed (e.g. an unauthenticated client reading
another tenant's data, an S3 user escalating to admin)
- A minimal, working reproduction or proof of concept
- Expected vs. actual behavior
A report without a working reproduction and a clear trust boundary is a
hardening suggestion, not a vulnerability. We are glad to receive those, but
they are handled on the normal issue tracker, not as security advisories.
### Automated and AI-assisted reports
Output from static analysis, dependency scanners, fuzzers, or LLMs is welcome
only when you have manually validated it and can supply a working reproduction
against a supported version, per the requirements above. Raw tool output,
speculative findings, or generated reports without a demonstrated exploit will
be closed as hardening suggestions.
## Trust model
SeaweedFS is built to run with its cluster components (master, volume servers,
and the raw filer API) on a trusted network. Those internal APIs are not an
authentication boundary unless you explicitly enable a control (for example
volume JWT or filer authentication) and that control is bypassed. Exposing an
internal port directly to untrusted clients is a deployment mistake, not a
vulnerability in SeaweedFS.
Reports are in scope when they cross a boundary SeaweedFS is meant to enforce,
for example:
- Unauthenticated access to data or operations that require authentication
- One S3 identity reading, writing, or deleting another identity's data
- Privilege escalation from a normal S3 user to administrative capability
- Bypass of Object Lock / retention where it is configured
- Remotely triggered data corruption or loss
Reports are generally out of scope when they require:
- Direct access to an internal cluster port that is meant to be private
- Full master, filer, or volume server access (already a full compromise)
- An insecure example configuration rather than a documented secure setup
- Local-only impact on a host the attacker already controls
## CVE assignment
When a report is confirmed, we publish an advisory and request the CVE through
GitHub. CVEs assigned by third parties without coordinating with us, or for
issues that do not cross a boundary described above, may be disputed.
## Response and disclosure
- We aim to acknowledge a valid report within a few business days.
- We will investigate, work on a fix, and coordinate a disclosure timeline
with you.
- Please allow time for a fix before any public disclosure.
-167
View File
@@ -1,167 +0,0 @@
# Design: Serializing Bucket Configuration Mutations
Issue #9651 — concurrent `PutBucketVersioning` + `PutBucketEncryption` (as Terraform
issues them in parallel) intermittently lose the encryption write.
## Root cause
The bucket's entire config lives in one filer entry, `/buckets/<name>`. Every
config API does a read-modify-write of that single entry, and the writes are not
serialized:
- `updateBucketConfig(bucket, fn)` (`s3api_bucket_config.go:468`) — sources from a
possibly-stale cached `BucketConfig`, mutates `Entry.Extended`, writes the
**whole** entry. Used by: versioning, object-lock config, lifecycle, ACL/owner.
- `UpdateBucketMetadata` → `setBucketMetadata` (`:1042`) — reads a fresh entry,
mutates `Entry.Content`, writes the **whole** entry. Used by: encryption, CORS,
tagging, ownership, policy, notification.
Two ingredients produce the lost update:
1. **No serialization** of the read→modify→write (the cache mutexes only guard the
in-memory map, not the RMW).
2. **Whole-entry rewrite from an independent snapshot** — `updateBucketConfig`
rebuilds from a stale cached `BucketConfig` whose `Content` predates the
concurrent encryption write, so writing the whole entry reverts `Content`.
Sequential calls always pass (each sees the previous write), so it only surfaces
under concurrency — and CI's slower IO widens the window (the "2 of ~12 runs").
## Goals
- No lost updates across concurrent bucket-config changes — for **all** config
fields, not just versioning/encryption.
- Correct for a single S3 gateway (the reported case) and for multiple gateways.
- Reuse the filer primitives just merged (per-path lock, `WriteCondition`,
`ObjectTransaction`); do not reintroduce a distributed lock.
- Minimal blast radius: the fix lands at the two chokepoint helpers.
## Non-goals
- Changing the one-entry-per-bucket storage model.
- Multi-filer-concurrent bucket writes (addressed only as an optional phase 3).
## The two ingredients map to two complementary fixes
### Fix A — serialize + read fresh (closes the window for whole-entry writers)
Both `updateBucketConfig` and `UpdateBucketMetadata` must run their RMW under one
per-bucket critical section, and **re-read the entry fresh from the filer inside
it** — not rebuild from the cached `BucketConfig`. The lock alone is insufficient:
without the fresh read, two serialized writers still each apply a stale snapshot.
### Fix B — field-level updates (removes the collision entirely)
The two writers touch disjoint fields (`Extended[versioning]` vs `Content`). If
each path updated only its own field instead of rewriting the whole entry, neither
could clobber the other regardless of ordering. This is the structural fix and
makes serialization a defense-in-depth concern rather than a correctness
requirement for cross-field cases.
## Where to serialize (layering)
The bucket entry is a single filer entry, so unlike object writes there is no
sharding — the question is purely the scope of the lock:
| Layer | Serializes across | Cost | Notes |
|---|---|---|---|
| 1. Gateway-local per-bucket lock | one gateway process | tiny | fixes the reported (single-gateway/CI) case |
| 2. Filer per-path lock via conditional write | all gateways on one filer | small | reuses #9640 `CreateEntry`+`WriteCondition` |
| 3. Route-by-key to bucket-key owner filer | all gateways and filers | medium | same mechanism as the object DLM-removal |
## Recommended plan (phased)
### Phase 1 — minimal fix for #9651 (gateway-local lock + fresh read)
Add a bounded per-bucket lock table to `S3ApiServer`, reusing the same
`util.LockTable` the filer uses for its per-path lock:
```go
// in S3ApiServer
bucketConfigLocks *util.LockTable[string] // serialize bucket-entry RMW
func (s3a *S3ApiServer) withBucketConfigLock(bucket string, fn func() s3err.ErrorCode) s3err.ErrorCode {
lk := s3a.bucketConfigLocks.AcquireLock("bucketConfig", bucket, util.ExclusiveLock)
defer s3a.bucketConfigLocks.ReleaseLock(bucket, lk)
return fn()
}
```
Wrap the RMW in **both** chokepoints, and inside the lock read the entry fresh:
- `updateBucketConfig`: acquire the lock; re-read `/buckets/<name>` from the filer
(not the cache); rebuild `BucketConfig` from that fresh entry; apply `fn`; write;
invalidate cache; release.
- `UpdateBucketMetadata`/`setBucketMetadata`: same lock key; it already reads fresh,
so it just needs to share the critical section.
Both must use the **same** lock keyed on `bucket`, so versioning and encryption
contend on one mutex. This closes the reported window. Limitation: only one
gateway; two gateways behind a load balancer still race.
Test: parallel `PutBucketVersioning` + `PutBucketEncryption`, assert both persist
(the exact Terraform scenario), plus an N-way parallel variant over distinct
fields.
### Phase 2 — robust across gateways (field-level + CAS via merged primitives)
Move the writers off whole-entry rewrites:
- **Extended-based config** (versioning, object-lock, ownership, tagging-in-Extended)
→ `ObjectTransaction` `PATCH_EXTENDED` on `/buckets/<name>`. The owner filer reads
the entry fresh under its per-path lock and merges only the named keys, so the
gateway never sends a whole-entry snapshot — this dissolves *both* ingredients for
these fields.
- **`Content`-based config** (encryption, CORS, tags blob) — **chosen and
implemented (b3): extend `PATCH_EXTENDED` with `set_content`.** Under the same
per-path lock the filer reads the entry fresh, merges extended attributes, and
replaces `Content`, preserving the rest. So a content write becomes a field-level
patch too — `setBucketMetadata` patches `Content`, `updateBucketConfig` patches
extended keys, and the two serialize on the lock instead of racing whole-entry
rewrites. This is cleaner than the alternatives below: no client-side retry, no
storage migration, and it reuses `ObjectTransaction`'s existing atomic lock.
- (b1, rejected) Conditional `CreateEntry` overwrite with `IF_ETAG_MATCH` + retry
(#9640): correct but needs client-side retry, and the bucket directory entry has
no reliable ETag to compare on.
- (b2, future) Migrate each per-feature config out of the single `Content` blob
into its own `Extended` key. Then even *intra-blob* writes (tags vs encryption)
stop racing. Larger migration; tracked separately.
Once all paths are field-level patches, the phase-1 gateway lock is unnecessary —
the filer enforces atomicity. (This is the path taken: phase 1 was skipped.)
### Phase 3 — multi-filer (only if needed)
If multiple filers can write `/buckets/<name>` concurrently, a filer-local per-path
lock no longer suffices. Route bucket-config writes to
`PrimaryForKey("/buckets/<name>")` (the lock-ring view) and serialize on that one
owner filer — the same route-by-key design used to take object writes off the DLM.
Overkill for rare config writes; include only if multi-filer bucket writes are real.
## Correctness summary
- Phase 1: all RMW for a bucket serialize within a gateway; the fresh read means the
second writer observes the first's change. Closes #9651 for single-gateway.
- Phase 2: `PATCH_EXTENDED` is atomic field-level merge at the filer (no snapshot);
CAS turns a concurrent `Content` write into a retry, enforced under the filer's
per-path lock — correct for any number of gateways sharing a filer.
- Phase 3: one owner filer serializes all writers — correct across filers too.
## Scope checklist (every path that RMWs the bucket entry)
All of these funnel through the two chokepoints, so fixing the chokepoints covers
them — but the fix must not leave any of them on an unserialized path:
- via `updateBucketConfig`: versioning, object-lock config, lifecycle, ACL/owner.
- via `UpdateBucketMetadata`/`setBucketMetadata`: encryption, CORS, tagging,
ownership controls, bucket policy, notification.
- bucket create/delete (`CreateEntry`/`DeleteEntry` of `/buckets/<name>`) already
go through the filer's per-path lock on `CreateEntry`; ensure they take the same
bucket lock if they also patch config.
## Cache rule (must document in code)
Under the lock, **read the entry from the filer, never rebuild from the cached
`BucketConfig`**. The cache is for reads; it must be invalidated on every write and
never be the source for an RMW. This is the single most important detail — the lock
without the fresh read does not fix the bug.
+3 -5
View File
@@ -2,15 +2,13 @@ FROM ubuntu:22.04
LABEL author="Chris Lu"
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
# Use faster mirrors and optimize package installation
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
# For testing purposes, running as root avoids permission complexities and dependency
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
apt-get -o Acquire::http::Timeout=15 update && \
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
--no-install-recommends \
--no-install-suggests \
curl \
+8 -28
View File
@@ -1,6 +1,4 @@
# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary,
# so arm64/arm/386 targets skip QEMU emulation of the whole compile.
FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder
FROM golang:1.25-alpine AS builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
ARG BRANCH=${BRANCH:-master}
@@ -14,28 +12,19 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
git checkout $BRANCH) || \
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
echo "Available branches:" && git branch -a && exit 1))
ARG TARGETOS TARGETARCH TARGETVARIANT
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& export GOOS=$TARGETOS GOARCH=$TARGETARCH \
&& case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \
&& CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed .
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
# Rust volume server: use pre-built binary from CI when available (placed in
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
# for non-native architectures.
# Rust volume server builder. Alpine packages avoid depending on the
# upstream rust:alpine manifest list, which no longer includes linux/386.
FROM alpine:3.23 as rust_builder
ARG TARGETARCH
ARG TAGS
COPY weed-volume-prebuilt/ /prebuilt/
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
WORKDIR /build/seaweed-volume
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
ARG TAGS
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
if [ "$TAGS" = "5BytesOffset" ]; then \
cargo build --release; \
@@ -47,10 +36,6 @@ RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
touch /weed-volume; \
fi
# Pre-built binaries arrive via GitHub Actions artifacts, which drop the
# executable bit, so the copied file is 0644 and exec fails with "Permission
# denied". Restore it (no-op for the empty placeholder, which stays size 0).
RUN chmod 0755 /weed-volume
FROM alpine AS final
LABEL author="Chris Lu"
@@ -65,8 +50,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
# To disable: docker run -e GODEBUG=fips140=off ...
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -98,8 +83,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -6
View File
@@ -7,7 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -40,8 +40,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -6
View File
@@ -34,7 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse snappy gflags curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -68,8 +68,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -1
View File
@@ -17,7 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
-6
View File
@@ -15,9 +15,3 @@ COPY tarantool /opt/tarantool/app
# build app
RUN tt build app
# the base image's default healthcheck assumes a single-instance layout and
# checks a control socket path that doesn't exist for this multi-instance
# app, so it never reports healthy; check instance status instead.
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=6 \
CMD tt status app | awk 'NR>2{if ($2!="RUNNING") exit 1} END{if (NR<=2) exit 1}'
-3
View File
@@ -127,9 +127,6 @@ test_tarantool: tags = tarantool
test_tarantool: build_tarantool_dev_env build
docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
test_keycloak_s3: build
docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
clean:
rm ./weed
@@ -1,53 +0,0 @@
{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "AKIAIOSFODNN7EXAMPLE",
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
},
{
"name": "steward",
"credentials": [
{
"accessKey": "steward-key",
"secretKey": "steward-secret"
}
],
"actions": [
"Read",
"List",
"Write"
]
},
{
"name": "le001",
"credentials": [
{
"accessKey": "le001-key",
"secretKey": "le001-secret"
}
],
"actions": [
"Read",
"List"
]
},
{
"name": "anonymous",
"actions": [
"Read"
]
}
]
}
-46
View File
@@ -1,46 +0,0 @@
services:
keycloak:
image: quay.io/keycloak/keycloak:26.0.7
command: ["start-dev", "--import-realm"]
environment:
KC_BOOTSTRAP_ADMIN_USERNAME: admin
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
KC_HTTP_ENABLED: "true"
KC_HOSTNAME: "http://keycloak:8080"
KC_HOSTNAME_STRICT: "false"
ports:
- "8080:8080"
volumes:
- ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
healthcheck:
test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
interval: 10s
timeout: 5s
retries: 5
start_period: 60s
s3:
image: chrislusf/seaweedfs:local
command:
- -v=9
- server
- -ip=s3
- -filer
- -master.volumeSizeLimitMB=16
- -volume
- -volume.max=0
- -volume.preStopSeconds=1
- -s3
- -s3.port=8333
- -s3.config=/etc/seaweedfs/s3_config.json
- -s3.iam.config=/etc/seaweedfs/iam_config.json
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volumes:
- ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
- ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
ports:
- "8333:8333"
depends_on:
keycloak:
condition: service_healthy
+2 -15
View File
@@ -37,21 +37,17 @@ if [ "$(id -u)" = "0" ]; then
fi
isArgPassed() {
# Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9
# library accepts both, and users may pick either form on the CLI.
arg="$1"
argWithEqualSign="$1="
argDouble="-$1"
argDoubleWithEqualSign="-$1="
shift
while [ $# -gt 0 ]; do
passedArg="$1"
shift
case $passedArg in
"$arg"|"$argDouble")
"$arg")
return 0
;;
"$argWithEqualSign"*|"$argDoubleWithEqualSign"*)
"$argWithEqualSign"*)
return 0
;;
esac
@@ -99,15 +95,6 @@ case "$1" in
exec /usr/bin/weed -logtostderr=true server $ARGS $@
;;
'mini')
ARGS="-dir=/data"
if isArgPassed "-dir" "$@"; then
ARGS=""
fi
shift
exec /usr/bin/weed -logtostderr=true mini $ARGS $@
;;
'filer')
ARGS=""
shift
+2 -2
View File
@@ -46,7 +46,7 @@ local filer_metadata = {
delete_by_directory_idx = function(directory)
local space = box.space.filer_metadata
local index = space.index.directory_idx
-- for each found directory
-- for each finded directories
for _, tuple in index:pairs({ directory }, { iterator = 'EQ' }) do
space:delete({ tuple[1], tuple[3] })
end
@@ -64,7 +64,7 @@ local filer_metadata = {
end
-- init results
local results = {}
-- for each found directory
-- for each finded directories
for _, tuple in directory_idx:pairs({ dirPath }, { iterator = 'EQ' }) do
-- filter by name
if filter_filename_func(tuple[3]) then
+203 -212
View File
@@ -1,11 +1,11 @@
module github.com/seaweedfs/seaweedfs
go 1.25.8
go 1.25.0
require (
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/pubsub v1.51.0
cloud.google.com/go/storage v1.64.0
cloud.google.com/go/pubsub v1.50.1
cloud.google.com/go/storage v1.60.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.55.8
github.com/beorn7/perks v1.0.1 // indirect
@@ -15,6 +15,7 @@ require (
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1
github.com/eapache/go-resiliency v1.6.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
@@ -25,32 +26,32 @@ require (
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-redsync/redsync/v4 v4.17.0
github.com/go-sql-driver/mysql v1.10.0
github.com/go-redsync/redsync/v4 v4.16.0
github.com/go-sql-driver/mysql v1.9.3
github.com/go-zookeeper/zk v1.0.4 // indirect
github.com/golang/protobuf v1.5.4
github.com/golang/snappy v1.0.0
github.com/google/btree v1.1.3
github.com/google/uuid v1.6.0
github.com/google/wire v0.7.0 // indirect
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
github.com/gorilla/mux v1.8.1
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/jackc/pgx/v5 v5.10.0
github.com/jackc/pgx/v5 v5.8.0
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
github.com/jinzhu/copier v0.4.0
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/klauspost/compress v1.19.1
github.com/klauspost/reedsolomon v1.14.1
github.com/klauspost/compress v1.18.5
github.com/klauspost/reedsolomon v1.13.3
github.com/kurin/blazer v0.5.3
github.com/linxGnu/grocksdb v1.10.8
github.com/mailru/easyjson v0.9.2 // indirect
github.com/mattn/go-isatty v0.0.23 // indirect
github.com/linxGnu/grocksdb v1.10.7
github.com/mailru/easyjson v0.9.1 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/olivere/elastic/v7 v7.0.32
@@ -59,15 +60,15 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/posener/complete v1.2.3
github.com/pquerna/cachecontrol v0.2.0
github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_model v0.6.2
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.2 // indirect
github.com/prometheus/procfs v0.20.1
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/seaweedfs/goexif v1.0.3
github.com/seaweedfs/raft v1.2.0
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/seaweedfs/raft v1.1.7
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/viper v1.21.0
@@ -85,81 +86,80 @@ require (
github.com/xdg-go/scram v1.2.0
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.etcd.io/etcd/client/v3 v3.6.12
go.etcd.io/etcd/client/v3 v3.6.9
go.mongodb.org/mongo-driver v1.17.9
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.46.0
gocloud.dev/pubsub/natspubsub v0.46.0
gocloud.dev/pubsub/rabbitpubsub v0.46.0
golang.org/x/crypto v0.54.0
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
golang.org/x/image v0.44.0
golang.org/x/net v0.57.0
gocloud.dev v0.45.0
gocloud.dev/pubsub/natspubsub v0.45.0
gocloud.dev/pubsub/rabbitpubsub v0.45.0
golang.org/x/crypto v0.49.0
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
golang.org/x/image v0.38.0
golang.org/x/net v0.51.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sys v0.47.0
golang.org/x/text v0.40.0 // indirect
golang.org/x/tools v0.48.0 // indirect
golang.org/x/sys v0.42.0
golang.org/x/text v0.35.0 // indirect
golang.org/x/tools v0.42.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.289.0
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect
google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6
google.golang.org/api v0.267.0
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/grpc v1.79.3
google.golang.org/protobuf v1.36.11
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/mathutil v1.7.1
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.53.0
modernc.org/sqlite v1.46.1
modernc.org/strutil v1.2.1
)
require (
cloud.google.com/go/kms v1.31.0
cloud.google.com/go/kms v1.26.0
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/Jille/raft-grpc-transport v1.6.1
github.com/ThreeDotsLabs/watermill v1.5.2
github.com/a-h/templ v0.3.1020
github.com/apache/cassandra-gocql-driver/v2 v2.1.2
github.com/apache/iceberg-go v0.6.0
github.com/ThreeDotsLabs/watermill v1.5.1
github.com/a-h/templ v0.3.977
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
github.com/apache/iceberg-go v0.5.0
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
github.com/arangodb/go-driver v1.6.9
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.43.2
github.com/aws/aws-sdk-go-v2/config v1.32.33
github.com/aws/aws-sdk-go-v2/credentials v1.19.32
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2
github.com/cognusion/imaging v1.0.4
github.com/aws/aws-sdk-go-v2 v1.41.5
github.com/aws/aws-sdk-go-v2/config v1.32.13
github.com/aws/aws-sdk-go-v2/credentials v1.19.13
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.44.1
github.com/go-ldap/ldap/v3 v3.4.13
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/hashicorp/raft v1.7.3
github.com/hashicorp/raft-boltdb/v2 v2.3.1
github.com/hashicorp/vault/api v1.23.0
github.com/hashicorp/vault/api v1.22.0
github.com/jhump/protoreflect v1.18.0
github.com/linkedin/goavro/v2 v2.15.0
github.com/mattn/go-sqlite3 v1.14.34
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.30.1
github.com/pkg/sftp v1.13.11
github.com/rabbitmq/amqp091-go v1.11.0
github.com/rclone/rclone v1.75.0
github.com/parquet-go/parquet-go v0.28.0
github.com/pkg/sftp v1.13.10
github.com/rabbitmq/amqp091-go v1.10.0
github.com/rclone/rclone v1.73.1
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.21.0
github.com/schollz/progressbar/v3 v3.19.1
github.com/seaweedfs/go-fuse/v2 v2.9.4
github.com/shirou/gopsutil/v4 v4.26.6
github.com/tarantool/go-option v1.1.0
github.com/tarantool/go-tarantool/v3 v3.0.0
github.com/testcontainers/testcontainers-go v0.43.0
github.com/redis/go-redis/v9 v9.18.0
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.2
github.com/shirou/gopsutil/v4 v4.26.2
github.com/tarantool/go-tarantool/v2 v2.4.2
github.com/testcontainers/testcontainers-go v0.40.0
github.com/tikv/client-go/v2 v2.0.7
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.146.3
go.etcd.io/etcd/client/pkg/v3 v3.6.12
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
go.etcd.io/etcd/client/pkg/v3 v3.6.9
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.22.0
golang.org/x/sync v0.20.0
golang.org/x/tools/godoc v0.1.0-deprecated
google.golang.org/grpc/security/advancedtls v1.0.0
)
@@ -170,28 +170,27 @@ require (
atomicgo.dev/cursor v0.2.0 // indirect
atomicgo.dev/keyboard v0.2.9 // indirect
atomicgo.dev/schedule v0.1.0 // indirect
cloud.google.com/go/longrunning v1.2.0 // indirect
cloud.google.com/go/pubsub/v2 v2.6.0 // indirect
cloud.google.com/go/longrunning v0.8.0 // indirect
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
github.com/ProtonMail/gopenpgp/v3 v3.4.1 // indirect
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 // indirect
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
github.com/adrg/xdg v0.5.3 // indirect
github.com/anchore/go-lzo v0.1.1 // indirect
github.com/anchore/go-lzo v0.1.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apache/arrow-go/v18 v18.7.0 // indirect
github.com/apache/thrift v0.24.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.5.2 // indirect
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
github.com/apache/thrift v0.22.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/bazelbuild/rules_go v0.46.0 // indirect
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
github.com/blevesearch/snowballstem v0.9.0 // indirect
github.com/boombuler/barcode v1.1.0 // indirect
github.com/buger/jsonparser v1.2.0 // indirect
github.com/buger/jsonparser v1.1.2 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
github.com/cockroachdb/errors v1.11.3 // indirect
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
@@ -201,52 +200,54 @@ require (
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v1.0.0-rc.4 // indirect
github.com/containerd/platforms v1.0.0-rc.1 // indirect
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/dave/dst v0.27.2 // indirect
github.com/diskfs/go-diskfs v1.9.4 // indirect
github.com/diskfs/go-diskfs v1.7.0 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/docker v28.5.2+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dromara/dongle v1.0.1 // indirect
github.com/gin-gonic/gin v1.11.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
github.com/go-git/go-billy/v5 v5.9.0 // indirect
github.com/go-git/go-billy/v5 v5.6.2 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/gookit/color v1.6.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gopherjs/gopherjs v1.17.2 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/hamba/avro/v2 v2.31.0 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2 // indirect
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jaegertracing/jaeger v1.47.0 // indirect
github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.12.0 // indirect
github.com/lib/pq v1.11.1 // indirect
github.com/lithammer/fuzzysearch v1.1.8 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/lpar/calendar v0.2.0 // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.2.0 // indirect
github.com/moby/moby/api v1.54.2 // indirect
github.com/moby/moby/client v0.4.0 // indirect
github.com/moby/patternmatcher v0.6.1 // indirect
github.com/moby/go-archive v0.1.0 // indirect
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/oklog/ulid/v2 v2.1.1 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
@@ -255,20 +256,18 @@ require (
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
github.com/pierrre/geohash v1.0.0 // indirect
github.com/pquerna/otp v1.5.0 // indirect
github.com/pterm/pterm v0.12.83 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/rclone/Proton-API-Bridge v1.0.4 // indirect
github.com/rclone/go-proton-api v1.0.3 // indirect
github.com/rogpeppe/go-internal v1.15.0 // indirect
github.com/pterm/pterm v0.12.82 // indirect
github.com/quic-go/quic-go v0.57.0 // indirect
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/sasha-s/go-deadlock v0.3.1 // indirect
github.com/smarty/assertions v1.15.0 // indirect
github.com/sony/gobreaker/v2 v2.4.0 // indirect
github.com/stretchr/objx v0.5.3 // indirect
github.com/substrait-io/substrait v0.87.0 // indirect
github.com/substrait-io/substrait-go/v8 v8.1.1 // indirect
github.com/substrait-io/substrait-protobuf/go v0.85.0 // indirect
github.com/twmb/avro v1.7.2 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/substrait-io/substrait v0.81.0 // indirect
github.com/substrait-io/substrait-go/v7 v7.4.0 // indirect
github.com/substrait-io/substrait-protobuf/go v0.81.0 // indirect
github.com/twpayne/go-geom v1.6.1 // indirect
github.com/twpayne/go-kml/v3 v3.2.1 // indirect
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
@@ -278,117 +277,122 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.uber.org/mock v0.5.2 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.38.0 // indirect
golang.org/x/mod v0.33.0 // indirect
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
gonum.org/v1/gonum v0.17.0 // indirect
)
require (
cel.dev/expr v0.25.2 // indirect
cloud.google.com/go/auth v0.20.0 // indirect
cel.dev/expr v0.25.1 // indirect
cloud.google.com/go/auth v0.18.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.11.0 // indirect
cloud.google.com/go/monitoring v1.29.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
github.com/Azure/go-ntlmssp v0.1.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
filippo.io/edwards25519 v1.1.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
github.com/Azure/go-ntlmssp v0.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
github.com/ProtonMail/go-crypto v1.4.1 // indirect
github.com/ProtonMail/go-crypto v1.3.0 // indirect
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
github.com/ProtonMail/go-srp v0.0.7 // indirect
github.com/PuerkitoBio/goquery v1.12.0 // indirect
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
github.com/PuerkitoBio/goquery v1.10.3 // indirect
github.com/abbot/go-http-auth v0.4.0 // indirect
github.com/andybalholm/brotli v1.2.2 // indirect
github.com/andybalholm/cascadia v1.3.4 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.33 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.33 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.33 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.34 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.33 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.33.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.45.2
github.com/aws/smithy-go v1.27.5
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
github.com/aws/smithy-go v1.24.2
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
github.com/buengese/sgzip v0.1.1 // indirect
github.com/calebcase/tmpfile v1.0.3 // indirect
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
github.com/cloudflare/circl v1.6.4 // indirect
github.com/cloudinary/cloudinary-go/v2 v2.16.0 // indirect
github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8 // indirect
github.com/cloudflare/circl v1.6.3 // indirect
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/creasty/defaults v1.8.0 // indirect
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0 // indirect
github.com/ebitengine/purego v0.10.1 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
github.com/geoffgarside/ber v1.2.0 // indirect
github.com/go-chi/chi/v5 v5.3.1 // indirect
github.com/go-chi/chi/v5 v5.2.5 // indirect
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/errors v0.22.8 // indirect
github.com/go-openapi/strfmt v0.27.0 // indirect
github.com/go-openapi/errors v0.22.4 // indirect
github.com/go-openapi/strfmt v0.25.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.3 // indirect
github.com/go-resty/resty/v2 v2.17.2 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goccy/go-json v0.10.6 // indirect
github.com/go-playground/validator/v10 v10.28.0 // indirect
github.com/go-resty/resty/v2 v2.16.5 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/gofrs/flock v0.13.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
github.com/gorilla/schema v1.4.1 // indirect
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/sessions v1.4.0
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -400,29 +404,30 @@ require (
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/k0kubun/pp v3.0.1+incompatible
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lanrat/extsort v1.4.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect
github.com/mattn/go-runewidth v0.0.24 // indirect
github.com/lpar/date v1.0.0 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nats.go v1.49.0 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
github.com/nats-io/nats.go v1.48.0 // indirect
github.com/nats-io/nkeys v0.4.12 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncw/swift/v2 v2.0.5 // indirect
@@ -430,100 +435,86 @@ require (
github.com/oklog/ulid v1.3.1 // indirect
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.121.0 // indirect
github.com/panjf2000/ants/v2 v2.12.1 // indirect
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
github.com/panjf2000/ants/v2 v2.11.3 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.27
github.com/pierrec/lz4/v4 v4.1.26
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pkg/xattr v0.4.12 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
github.com/relvacode/iso8601 v1.7.0 // indirect
github.com/rfjakob/eme v1.2.0 // indirect
github.com/rfjakob/eme v1.1.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/samber/lo v1.53.0 // indirect
github.com/samber/lo v1.52.0 // indirect
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
github.com/sony/gobreaker v1.0.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
github.com/tarantool/go-iproto v1.1.0 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect
github.com/tinylib/msgp v1.5.0 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/twmb/murmur3 v1.1.8 // indirect
github.com/unknwon/goconfig v1.0.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260428144813-1c07baab7f7b // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zeebo/errs v1.4.0 // indirect
go.etcd.io/bbolt v1.5.0 // indirect
go.etcd.io/etcd/api/v3 v3.6.12 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.etcd.io/etcd/api/v3 v3.6.9 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
go.opentelemetry.io/otel v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
go.opentelemetry.io/otel/trace v1.40.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
golang.org/x/term v0.45.0
golang.org/x/time v0.15.0
google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d // indirect
golang.org/x/term v0.41.0 // indirect
golang.org/x/time v0.14.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.73.4 // indirect
modernc.org/libc v1.68.0 // indirect
moul.io/http2curl/v2 v2.3.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
storj.io/common v0.0.0-20260629224719-ba1bff0a7846 // indirect
storj.io/drpc v1.0.0 // indirect
storj.io/eventkit v0.0.0-20260716074419-6861a92e2aa5 // indirect
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
storj.io/infectious v0.0.2 // indirect
storj.io/picobuf v0.0.4 // indirect
storj.io/uplink v1.14.3 // indirect
storj.io/uplink v1.13.1 // indirect
)
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
// apache/thrift v0.23.0 fixes CVE-2026-41602 but compares int against the
// untyped math.MaxUint32 in lib/go/thrift/framed_transport.go, which overflows
// int on 32-bit GOARCHes (e.g. openbsd/arm, linux/arm) and fails to compile.
// Upstream fixed the range check post-release; pin to that commit until the
// next tagged release carries both the CVE fix and the 32-bit fix.
replace github.com/apache/thrift => github.com/apache/thrift v0.23.1-0.20260429145742-d2acd3c49e58
// tyler-smith/go-bip39 was deleted from GitHub, so `go mod download` fails for
// anyone resolving it directly (GOPROXY=direct). It only reaches us transitively
// through rclone's internxt backend, which calls IsMnemonicValid and NewSeed.
// cosmos/go-bip39 is a maintained, API-compatible fork.
replace github.com/tyler-smith/go-bip39 => github.com/cosmos/go-bip39 v1.0.0
+575 -513
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -6,7 +6,7 @@
# Usage:
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
# curl -fsSL ... | bash -s -- --version v3.93 --dir /usr/local/bin
#
# Options:
# --component COMP Which binary to install: weed, volume-rust, all (default: weed)
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.41"
appVersion: "4.17"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.41.0
version: 4.17.0
+1 -71
View File
@@ -49,35 +49,6 @@ CREATE TABLE IF NOT EXISTS `filemeta` (
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
#### RocksDB variant
The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
To use this image with the Helm chart, override the image on all three components and disable
the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
override the image's built-in RocksDB configuration:
```yaml
# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
master:
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
volume:
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
filer:
enablePVC: true
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
extraEnvironmentVars:
WEED_LEVELDB2_ENABLED: "false"
```
Notes:
* `master` and `volume` use the same image tag so that all components share a consistent
SeaweedFS build; RocksDB itself is only used by the filer.
* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
### Node Labels
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
@@ -365,47 +336,6 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum
helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
```
## Network Policies
In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
```bash
helm install seaweedfs seaweedfs/seaweedfs --set networkPolicy.enabled=true
```
That alone leaves outbound traffic untouched, which is enough when the namespace's default-deny only restricts ingress. If it lists `Egress` in its `policyTypes` too - the usual baseline - the components still cannot resolve DNS, and you need the second opt-in below as well.
Egress is separate because the chart knows where its own components live but not where your filer store, notification sink or remote tier does, and because in a namespace with no default-deny at all, adding egress rules would narrow the components from "may reach anything" to "may reach these peers":
```yaml
networkPolicy:
enabled: true
egress:
enabled: true
kubeApiServer:
# the endpoint behind the kubernetes service, not its ClusterIP
cidrs: ["172.18.0.2/32"]
extraEgress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgresql
ports:
- protocol: TCP
port: 5432
```
`kubeApiServer.cidrs` is only demanded when something in the release actually needs the API server, which is the COSI sidecar and, on an upgrade that grows a volume PVC, the resize hook. No seaweedfs component itself speaks to it.
Anything reaching the release from outside - an ingress controller, a Prometheus in another namespace - goes into `networkPolicy.extraIngress`, or into `networkPolicy.components.<component>.extraIngress` for a single component. See the `networkPolicy` block in `values.yaml` for the full set.
Two things worth knowing before you turn this on:
- **Monitoring stops.** The metrics ports are admitted from release pods like every other port, so with `global.seaweedfs.monitoring.enabled` the ServiceMonitors keep scraping targets a Prometheus in another namespace can no longer reach. Nothing reports it; add the scraper's namespace to `extraIngress`.
- **The resize hook's policy is a Helm hook.** Its Job runs before the release manifest is applied, so the policy has to be a `pre-install` hook too. Helm does not garbage-collect hook resources, so on an upgrade that grows a volume PVC the policy is created and then left behind on uninstall - delete `<release>-seaweedfs-volume-resize-hook` by hand if it bothers you.
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
## OpenShift Support
SeaweedFS can be deployed on OpenShift or any cluster enforcing the Kubernetes "restricted" Pod Security Standard. By default, OpenShift blocks containers that run as root or use `hostPath` volumes.
@@ -425,4 +355,4 @@ helm install seaweedfs seaweedfs/seaweedfs \
## Enterprise
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.
which has a self-healing storage format with better data protection.
@@ -1 +0,0 @@
# Default configuration, kept so chart-testing still installs the chart as-is.
-5
View File
@@ -1,5 +0,0 @@
# SFTP install: the pod only becomes ready if the server loads a host key
# from the generated sftp-ssh-secret, so this exercises the whole path.
sftp:
enabled: true
enableAuth: true
@@ -3666,291 +3666,6 @@
],
"title": "S3 Bucket Object Count",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 130
},
"id": 92,
"panels": [],
"title": "Filer Object Size Distribution",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Rate of new objects created, split by size range.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "objects/s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 25,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 131
},
"id": 93,
"links": [],
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.1.2",
"targets": [
{
"exemplar": true,
"expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval]))",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "< 1KB",
"refId": "A",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1KB - 100KB",
"refId": "B",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100KB - 1MB",
"refId": "C",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1MB - 100MB",
"refId": "D",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100MB - 1GB",
"refId": "E",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "> 1GB",
"refId": "F",
"step": 60
}
],
"title": "Filer Object Write Rate by Size Range",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Objects created per size range over the selected time window.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 131
},
"id": 94,
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"valueMode": "color"
},
"pluginVersion": "8.1.2",
"targets": [
{
"exemplar": true,
"expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range]))",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "< 1KB",
"refId": "A"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1KB - 100KB",
"refId": "B"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100KB - 1MB",
"refId": "C"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1MB - 100MB",
"refId": "D"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100MB - 1GB",
"refId": "E"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "> 1GB",
"refId": "F"
}
],
"title": "Filer Object Size Distribution",
"type": "bargauge"
}
],
"refresh": "",
@@ -15,26 +15,15 @@ metadata:
{{- toYaml .Values.admin.service.annotations | nindent 4 }}
{{- end }}
spec:
{{- $serviceType := .Values.admin.service.type }}
{{- $nodePorts := dict }}
{{- if or (eq $serviceType "NodePort") (eq $serviceType "LoadBalancer") }}
{{- $nodePorts = .Values.admin.service.nodePorts | default dict }}
{{- end }}
type: {{ $serviceType }}
type: {{ .Values.admin.service.type }}
ports:
- name: "http"
port: {{ .Values.admin.port }}
targetPort: {{ .Values.admin.port }}
{{- if $nodePorts.http }}
nodePort: {{ $nodePorts.http }}
{{- end }}
protocol: TCP
- name: "grpc"
port: {{ .Values.admin.grpcPort }}
targetPort: {{ .Values.admin.grpcPort }}
{{- if $nodePorts.grpc }}
nodePort: {{ $nodePorts.grpc }}
{{- end }}
protocol: TCP
selector:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

Some files were not shown because too many files have changed in this diff Show More