mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-13 01:55:50 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Chris Lu
|
a4bf9fe47f | ||
|
Chris Lu
|
b3ea57b5d7 |
@@ -1,22 +0,0 @@
|
||||
[codespell]
|
||||
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
|
||||
skip = .git,.git-meta,.gitignore,.gitattributes,*.svg,go.sum,vendor,*.lock,*.css,*.min.*,.codespellrc,*_templ.go
|
||||
check-hidden = true
|
||||
# Ignore camelCase and PascalCase identifiers (very common in Go/Rust/JS
|
||||
# source, e.g. allLocations, publishErr, ReadInside, FlushInterval).
|
||||
ignore-regex = \b[a-z]+[A-Z]\w*\b|\b[A-Z][a-z]+[A-Z]\w*\b
|
||||
# visibles: variable name for VisibleInterval collections in filer/mount code
|
||||
# fo: `*FilerOptions` receiver name (e.g. `func (fo *FilerOptions) ...`)
|
||||
# te: "truncate error" local variable (e.g. `if te := w.Truncate(end); te != nil`)
|
||||
# ser: Rust serde serializer variable (`serde_json::ser`, `let mut ser = ...`)
|
||||
# bject: intentional wildcard test data (e.g. `s3:Get?bject` matching `s3:GetObject`)
|
||||
# unparseable: accepted alternate spelling used throughout the codebase
|
||||
# keep-alives: correct plural of the technical term (SSH/HTTP keep-alive)
|
||||
# tread: valid English word in the idiom "tread carefully" (help text)
|
||||
# anc: variable abbreviation for "ancestor" in tree/path tests
|
||||
# ue: appears inside JSON test fixtures with embedded escaped quotes (Bl\"ue)
|
||||
# auther: local variable meaning "authenticator" (tls.go: `auther := Authenticator{}`)
|
||||
# thirdparty: literal Maven groupId `org.apache.hadoop.thirdparty` (external, cannot rename)
|
||||
# unknwon: GitHub username / Go module path (`github.com/unknwon/goconfig`)
|
||||
# atleast: CLI mode literal string in test/benchmark/fuse_db/bin/sqlite_verify.py
|
||||
ignore-words-list = visibles,fo,te,ser,bject,unparseable,keep-alives,tread,anc,ue,auther,thirdparty,unknwon,atleast
|
||||
@@ -1,66 +0,0 @@
|
||||
name: Fix fusermount3 setuid
|
||||
description: >
|
||||
Make sure the fusermount3 an unprivileged mount will find can actually mount.
|
||||
Some runner images carry a second, source-built fusermount3 in /usr/local/bin
|
||||
that shadows the distro one in PATH; it is neither setuid nor root-owned, so
|
||||
every mount fails with "mount failed: Operation not permitted". Both the Go
|
||||
mount and sw-fuse resolve the helper through PATH.
|
||||
Run it after the step that apt-installs fuse3.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Point PATH at a fusermount3 that can mount
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# setuid only grants root when root owns the file, and exec follows
|
||||
# symlinks, so judge the target.
|
||||
can_mount() {
|
||||
[ -u "$1" ] && [ "$(stat -Lc %u "$1")" = 0 ]
|
||||
}
|
||||
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
echo "no fusermount3 in PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
if can_mount "$bin"; then
|
||||
ls -l "$bin"
|
||||
exit 0
|
||||
fi
|
||||
echo "$bin cannot mount unprivileged:"
|
||||
ls -l "$bin"
|
||||
|
||||
# The distro fusermount3 is setuid root and is the one meant to be used.
|
||||
# Reach it through a symlink earlier in PATH: exec resolves the link, so
|
||||
# the target keeps its setuid bit, and nothing on the image is modified.
|
||||
for distro in /usr/bin/fusermount3 /bin/fusermount3; do
|
||||
if [ "$distro" != "$bin" ] && can_mount "$distro"; then
|
||||
mkdir -p "$RUNNER_TEMP/fuse-bin"
|
||||
ln -sf "$distro" "$RUNNER_TEMP/fuse-bin/fusermount3"
|
||||
echo "$RUNNER_TEMP/fuse-bin" >> "$GITHUB_PATH"
|
||||
echo "using $distro instead"
|
||||
ls -l "$distro"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
# No usable distro binary, so setting the bit is the only way out - but
|
||||
# the target came from PATH: do that only for a root-owned system binary,
|
||||
# never for anything else that happens to sit there.
|
||||
case "$bin" in
|
||||
/bin/*|/sbin/*|/usr/bin/*|/usr/sbin/*|/usr/local/bin/*|/usr/local/sbin/*) ;;
|
||||
*) echo "refusing to setuid $bin: outside the system bin paths" >&2; exit 1 ;;
|
||||
esac
|
||||
if [ -L "$bin" ] || [ ! -f "$bin" ] || [ ! -x "$bin" ]; then
|
||||
echo "refusing to setuid $bin: not a regular executable file" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(stat -c %u "$bin")" != 0 ] || [ "$(stat -c %g "$bin")" != 0 ]; then
|
||||
echo "refusing to setuid $bin: not owned by root:root" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo chmod u+s "$bin"
|
||||
ls -l "$bin"
|
||||
@@ -3,11 +3,6 @@ name: "go: build dev binaries"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/binaries_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
@@ -48,7 +43,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
@@ -97,7 +92,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -2,11 +2,6 @@ name: "Code Scanning - Action"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/codeql.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/codeql
|
||||
@@ -23,11 +18,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.37.6
|
||||
uses: github/codeql-action/init@v4
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -35,7 +30,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4.37.6
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -49,4 +44,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4.37.6
|
||||
uses: github/codeql-action/analyze@v4
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
# Codespell configuration is within .codespellrc
|
||||
---
|
||||
name: Codespell
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
branches: [master]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
codespell:
|
||||
name: Check for spelling errors
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Codespell
|
||||
uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
|
||||
@@ -3,21 +3,13 @@ name: "docker: build dev containers"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-worker/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/container_dev.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
@@ -30,7 +22,10 @@ jobs:
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -50,29 +45,16 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
protoc=$(find ~/.cargo/registry/src -path '*protoc-bin-vendored-linux-x86_64*/bin/protoc' | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
@@ -81,19 +63,11 @@ jobs:
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
@@ -101,12 +75,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
@@ -120,16 +94,7 @@ jobs:
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -146,7 +111,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.2.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
@@ -163,14 +128,14 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -58,7 +58,7 @@ jobs:
|
||||
sudo ldconfig
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -126,31 +126,27 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.2.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Determine branch to build
|
||||
id: branch
|
||||
env:
|
||||
INPUT_REF: ${{ inputs.seaweedfs_ref }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
if [ -n "$INPUT_REF" ]; then
|
||||
echo "branch=$INPUT_REF" >> "$GITHUB_OUTPUT"
|
||||
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
|
||||
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo "branch=$HEAD_REF" >> "$GITHUB_OUTPUT"
|
||||
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT"
|
||||
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
@@ -72,10 +72,13 @@ jobs:
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
@@ -94,29 +97,16 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
protoc=$(find ~/.cargo/registry/src -path '*protoc-bin-vendored-linux-x86_64*/bin/protoc' | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
@@ -133,20 +123,13 @@ jobs:
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
@@ -158,7 +141,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Free Disk Space
|
||||
@@ -194,7 +177,7 @@ jobs:
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
@@ -208,16 +191,7 @@ jobs:
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -233,7 +207,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v4.2.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
@@ -247,13 +221,13 @@ jobs:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -301,21 +275,21 @@ jobs:
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
@@ -333,16 +307,7 @@ jobs:
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
@@ -402,7 +367,7 @@ jobs:
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4.37.6
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
@@ -443,7 +408,7 @@ jobs:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
@@ -465,12 +430,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -43,14 +43,14 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.2.0
|
||||
uses: docker/setup-qemu-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -33,7 +33,6 @@ permissions:
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
IMAGE: ghcr.io/chrislusf/seaweedfs
|
||||
|
||||
# Limit concurrent builds to avoid rate limits
|
||||
concurrency:
|
||||
@@ -42,10 +41,9 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
# The volume server and the Rust maintenance worker, cross-compiled for
|
||||
# amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
|
||||
# ~15 minutes of native compilation.
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
|
||||
# emulated cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
@@ -58,7 +56,10 @@ jobs:
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -78,29 +79,16 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
protoc=$(find ~/.cargo/registry/src -path '*protoc-bin-vendored-linux-x86_64*/bin/protoc' | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
@@ -117,66 +105,73 @@ jobs:
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
|
||||
# One job per (variant, platform) on a native runner, pushed by digest;
|
||||
# the merge job stitches the digests into one multi-arch tag.
|
||||
# ── Build Docker containers ─────────────────────────────────────────
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# Build sequentially to avoid rate limits
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
include:
|
||||
# Normal volume - multi-arch
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- variant: normal
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk - multi-arch
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
rust_variant: large-disk
|
||||
|
||||
# Full tags - multi-arch
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- variant: large_disk_full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
rust_variant: large-disk
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
rust_variant: large-disk
|
||||
|
||||
steps:
|
||||
- name: Skip unselected variant
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant
|
||||
run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0
|
||||
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
@@ -191,45 +186,41 @@ jobs:
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
sudo docker system prune -af --volumes
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
|
||||
|
||||
- name: Docker meta
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.IMAGE }}
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
|
||||
- name: Set up QEMU
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu
|
||||
uses: docker/setup-qemu-action@v4.2.0
|
||||
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -237,111 +228,91 @@ jobs:
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build and push ${{ matrix.variant }} (${{ matrix.platform }})
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
platforms: ${{ matrix.platforms }}
|
||||
# Push to GHCR to avoid Docker Hub rate limits on pulls
|
||||
tags: |
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
# Flat single-platform manifest so imagetools create assembles cleanly.
|
||||
provenance: false
|
||||
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
# max only for rocksdb: its RocksDB compile is sha-independent and worth
|
||||
# keeping; go-build layers are sha-busted every release, so min elsewhere.
|
||||
cache-to: type=gha,mode=${{ matrix.variant == 'rocksdb' && 'max' || 'min' }},scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
|
||||
build-args: |
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
|
||||
|
||||
- name: Export digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
|
||||
- name: Clean up build artifacts
|
||||
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
sudo docker system prune -f
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
- name: Upload digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: digest-${{ matrix.variant }}-${{ matrix.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Assemble each variant's per-platform digests into one tag, then mirror to Docker Hub.
|
||||
merge:
|
||||
needs: [build]
|
||||
copy-to-dockerhub:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant: [normal, large_disk, full, large_disk_full, rocksdb]
|
||||
include:
|
||||
- { variant: normal, tag_suffix: "" }
|
||||
- { variant: large_disk, tag_suffix: _large_disk }
|
||||
- { variant: full, tag_suffix: _full }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full }
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb }
|
||||
- variant: normal
|
||||
tag_suffix: ""
|
||||
- variant: large_disk
|
||||
tag_suffix: _large_disk
|
||||
- variant: full
|
||||
tag_suffix: _full
|
||||
- variant: large_disk_full
|
||||
tag_suffix: _large_disk_full
|
||||
- variant: rocksdb
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
|
||||
steps:
|
||||
- name: Download digests
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
pattern: digest-${{ matrix.variant }}-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Create multi-arch tag on GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
|
||||
docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
|
||||
- name: Install crane
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -349,37 +320,45 @@ jobs:
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Copy ${{ matrix.variant }} to Docker Hub
|
||||
|
||||
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
# Function to retry with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
|
||||
# Copy multi-arch image from GHCR to Docker Hub with retry
|
||||
# This is much more efficient than pulling/pushing individual arch images
|
||||
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
|
||||
retry_with_backoff crane copy \
|
||||
${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
echo "Copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
@@ -388,7 +367,7 @@ jobs:
|
||||
# CVE, base-image lag).
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
needs: [build]
|
||||
if: github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
@@ -401,7 +380,7 @@ jobs:
|
||||
variant: large_disk
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -415,7 +394,7 @@ jobs:
|
||||
# Scan the multi-arch tag on GHCR (already pushed by the build job).
|
||||
# Trivy scans the runner's native platform; OS packages are identical
|
||||
# across architectures since they all share the same alpine base.
|
||||
image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
@@ -427,7 +406,7 @@ jobs:
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4.37.6
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
@@ -439,7 +418,7 @@ jobs:
|
||||
# trivy-scan: vuln findings are reported but do not block `latest`.
|
||||
tag-latest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -450,13 +429,13 @@ jobs:
|
||||
latest_tag: latest_large_disk
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -496,24 +475,24 @@ jobs:
|
||||
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
|
||||
DST_TAG="${{ matrix.latest_tag }}"
|
||||
|
||||
echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}"
|
||||
echo "Tagging ghcr.io/chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "ghcr.io/chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
@@ -82,13 +82,13 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v1
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v1
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -21,22 +21,19 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'telemetry/server/go.mod'
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
run: |
|
||||
# telemetry/server is its own Go module; build from within it
|
||||
cd telemetry/server
|
||||
go mod tidy
|
||||
echo "Building telemetry server..."
|
||||
GOOS=linux GOARCH=amd64 go build -o ../../telemetry-server .
|
||||
cd ../..
|
||||
GOOS=linux GOARCH=amd64 go build -o telemetry-server ./telemetry/server/main.go
|
||||
ls -la telemetry-server
|
||||
echo "Build completed successfully"
|
||||
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "End to End"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/e2e
|
||||
@@ -36,23 +24,18 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
|
||||
@@ -61,10 +44,13 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# Use faster mirrors and install with timeout
|
||||
sudo rm -f /etc/apt/sources.list.d/azure-cli.list /etc/apt/sources.list.d/microsoft-prod.list
|
||||
# Same helper the e2e image installs through: the runner's own list is
|
||||
# azure-only too, and an outage there fails this step outright.
|
||||
sudo ./apt-install fuse
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee /etc/apt/sources.list
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
|
||||
|
||||
sudo apt-get update --fix-missing
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends fuse
|
||||
|
||||
# Verify FUSE installation
|
||||
echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')"
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "EC Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -28,13 +16,13 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
@@ -43,10 +31,7 @@ jobs:
|
||||
- name: Run EC Integration Tests
|
||||
working-directory: test/erasure_coding
|
||||
run: |
|
||||
# The suite now includes the interruption matrix and runs close to Go's
|
||||
# default 10m binary timeout on slower runners; bound it by the job's
|
||||
# 30m budget instead.
|
||||
go test -v -timeout 25m
|
||||
go test -v
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
|
||||
@@ -8,7 +8,6 @@ on:
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
@@ -16,7 +15,6 @@ on:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
@@ -33,24 +31,20 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Configure FUSE
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
name: "FUSE Volume Server Failover Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/operation/upload_content.go'
|
||||
- 'test/fuse_failover/**'
|
||||
- '.github/workflows/fuse-failover.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/operation/upload_content.go'
|
||||
- 'test/fuse_failover/**'
|
||||
- '.github/workflows/fuse-failover.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-failover
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-failover:
|
||||
name: FUSE Volume Server Failover
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run failover integration tests
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=30m ./test/fuse_failover/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-failover-test-logs
|
||||
path: /tmp/seaweedfs-fuse-failover-logs/
|
||||
retention-days: 3
|
||||
@@ -7,14 +7,12 @@ on:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/fuse-integration
|
||||
@@ -31,17 +29,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Configure FUSE
|
||||
- name: Install FUSE and dependencies
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed, and
|
||||
# go-fuse is pure Go, so the libfuse headers were never linked against.
|
||||
sudo apt-get update
|
||||
# fuse3 is pre-installed on ubuntu-22.04 runners and conflicts
|
||||
# with the legacy fuse package, so only install the dev headers.
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
# Allow non-root FUSE mounts with allow_other
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
@@ -49,9 +49,6 @@ jobs:
|
||||
fusermount3 --version || fusermount --version || true
|
||||
ls -la /dev/fuse
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
|
||||
@@ -11,7 +11,6 @@ on:
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
@@ -22,7 +21,6 @@ on:
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
@@ -39,24 +37,20 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Configure FUSE
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
|
||||
@@ -3,18 +3,8 @@ name: "go: build binary"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/go
|
||||
@@ -30,9 +20,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Get dependencies
|
||||
@@ -47,104 +37,28 @@ jobs:
|
||||
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
|
||||
if grep -q "vet:" vet-output.txt; then exit 1; fi
|
||||
|
||||
vet-32bit:
|
||||
name: Go Vet 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
|
||||
run: |
|
||||
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
|
||||
if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
build-cross:
|
||||
name: Build cross-platform (${{ matrix.goos }}/${{ matrix.goarch }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# One target's breakage should not hide the other three.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { goos: windows, goarch: amd64 }
|
||||
- { goos: windows, goarch: arm64 }
|
||||
- { goos: freebsd, goarch: amd64 }
|
||||
- { goos: darwin, goarch: arm64 }
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
# Nothing else on a PR compiles these, so per-OS syscall constants creep
|
||||
# back into shared files unnoticed and only a release build catches it.
|
||||
- name: Cross-compile
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
run: |
|
||||
go build ./weed/...
|
||||
# Tests too: they reach for per-OS syscall constants the build does
|
||||
# not, and only compiling them catches an untagged one.
|
||||
go vet ./weed/mount/... ./weed/command/... 2>&1 |
|
||||
grep -v "MessageState contains sync.Mutex" | tee /tmp/vet.txt
|
||||
if grep -q "vet:" /tmp/vet.txt; then exit 1; fi
|
||||
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
redis:
|
||||
image: redis:8
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Test
|
||||
env:
|
||||
RUN_REDIS_TESTS: "1"
|
||||
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
|
||||
|
||||
test-32bit:
|
||||
name: Test 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
# 386 test binaries run natively on the amd64 runner. This catches what vet
|
||||
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
- name: Test linux/386
|
||||
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
|
||||
|
||||
+25
-1281
File diff suppressed because it is too large
Load Diff
@@ -15,7 +15,7 @@ jobs:
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
|
||||
@@ -25,10 +25,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
name: "release: java clients"
|
||||
|
||||
# Publishes the SeaweedFS Java clients (seaweedfs-client and
|
||||
# seaweedfs-hadoop3-client) to Maven Central via the Sonatype Central Portal.
|
||||
#
|
||||
# Required repository secrets:
|
||||
# MAVEN_CENTRAL_USERNAME - Central Portal user token username (central.sonatype.com -> Account -> Generate User Token)
|
||||
# MAVEN_CENTRAL_PASSWORD - Central Portal user token password
|
||||
# MAVEN_GPG_PRIVATE_KEY - ASCII-armored signing key:
|
||||
# gpg --homedir <keyring> --armor --export-secret-keys <KEYID> > key.asc
|
||||
# MAVEN_GPG_PASSPHRASE - passphrase for that key
|
||||
#
|
||||
# A plain run publishes to Central. Check dry_run to exercise secrets, GPG
|
||||
# signing, and the build without committing or publishing.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Version to publish, e.g. 4.39. Leave blank to use the current SeaweedFS version."
|
||||
type: string
|
||||
required: false
|
||||
dry_run:
|
||||
description: "Build and GPG-sign only; skip the version commit and the Central upload"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: java-release
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
# Java 17 needs these opens for the Maven publishing plugins' reflective access.
|
||||
MAVEN_OPTS: >-
|
||||
--add-opens=java.base/java.util=ALL-UNNAMED
|
||||
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
|
||||
--add-opens=java.base/java.text=ALL-UNNAMED
|
||||
--add-opens=java.desktop/java.awt.font=ALL-UNNAMED
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish to Maven Central
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# Full history so the version-bump commit can rebase onto a moved master.
|
||||
fetch-depth: 0
|
||||
|
||||
# An explicit input wins; otherwise fall back to the current SeaweedFS
|
||||
# version (MAJOR.MINOR) from constants.go, matching its %d.%02d formatting.
|
||||
- name: Resolve version
|
||||
id: resolve
|
||||
env:
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
VERSION="$INPUT_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
CONST=weed/util/version/constants.go
|
||||
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
|
||||
echo "No version given; using current SeaweedFS version ${VERSION}."
|
||||
fi
|
||||
if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR, e.g. 4.39 (got '$VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
server-id: central
|
||||
server-username: MAVEN_CENTRAL_USERNAME
|
||||
server-password: MAVEN_CENTRAL_PASSWORD
|
||||
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
# client carries the version literally; hdfs3 derives its own version and
|
||||
# its seaweedfs-client dependency from the seaweedfs.client.version property.
|
||||
- name: Set versions in poms
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
V=org.codehaus.mojo:versions-maven-plugin:2.16.2
|
||||
mvn -B -ntp -f other/java/client/pom.xml "${V}:set" \
|
||||
-DnewVersion="$VERSION" -DgenerateBackupPoms=false
|
||||
mvn -B -ntp -f other/java/hdfs3/pom.xml "${V}:set-property" \
|
||||
-Dproperty=seaweedfs.client.version -DnewVersion="$VERSION" -DgenerateBackupPoms=false
|
||||
|
||||
- name: Commit version bump
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet; then
|
||||
echo "Poms already at ${VERSION}; nothing to commit."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add other/java/client/pom.xml other/java/hdfs3/pom.xml
|
||||
git commit -m "java ${VERSION}"
|
||||
# Rebase and retry so a concurrent push to master doesn't lose the bump.
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if git push origin HEAD:master; then
|
||||
exit 0
|
||||
fi
|
||||
echo "push rejected (attempt ${attempt}); rebasing onto latest master"
|
||||
git pull --rebase origin master
|
||||
done
|
||||
echo "::error::could not push version bump after retries"
|
||||
exit 1
|
||||
|
||||
# Tests are skipped here: the integration tests need a live filer, and the
|
||||
# offline unit tests already run on every push in java_unit_tests.yml.
|
||||
# dry_run stops at `install` (build + GPG sign, no upload). A real run deploys.
|
||||
# client goes first either way so its install populates the local repo for hdfs3.
|
||||
- name: Publish seaweedfs-client
|
||||
working-directory: other/java/client
|
||||
env:
|
||||
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
|
||||
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
|
||||
mvn -B -ntp $GOAL -DskipTests
|
||||
|
||||
- name: Publish seaweedfs-hadoop3-client
|
||||
working-directory: other/java/hdfs3
|
||||
env:
|
||||
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
|
||||
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
|
||||
mvn -B -ntp $GOAL -DskipTests
|
||||
|
||||
# Bump every version the wiki quotes for the Java clients. Runs only after
|
||||
# a successful real publish. Uses GITHUB_TOKEN; set a WIKI_TOKEN secret if
|
||||
# the token can't push to the wiki.
|
||||
- name: Update wiki client versions
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WIKI_TOKEN: ${{ secrets.WIKI_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TOKEN="${WIKI_TOKEN:-$GH_TOKEN}"
|
||||
git clone -q "https://x-access-token:${TOKEN}@github.com/${{ github.repository }}.wiki.git" wiki
|
||||
cd wiki
|
||||
for f in $(grep -rlE "seaweedfs-(client|hadoop3-client)" --include="*.md" .); do
|
||||
NEWV="$VERSION" perl -0777 -i -pe '
|
||||
my $v = $ENV{NEWV};
|
||||
s{(<artifactId>seaweedfs-(?:client|hadoop3-client)</artifactId>\s*<version>)[0-9]+\.[0-9]+(</version>)}{$1$v$2}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client):)[0-9]+\.[0-9]+}{$1$v}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client)-)[0-9]+\.[0-9]+(\.jar)}{$1$v$2}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client)/)[0-9]+\.[0-9]+(/)}{$1$v$2}g;
|
||||
' "$f"
|
||||
done
|
||||
if git diff --quiet; then
|
||||
echo "Wiki already references ${VERSION}."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git commit -am "java clients ${VERSION}"
|
||||
git push
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v5
|
||||
|
||||
@@ -3,24 +3,8 @@ name: "Kafka Quick Test (Load Test with Schema Registry)"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
concurrency:
|
||||
@@ -37,22 +21,17 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
|
||||
@@ -3,24 +3,8 @@ name: "Kafka Gateway Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/kafka-tests
|
||||
@@ -43,7 +27,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [unit-tests-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -51,13 +35,13 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Container Environment
|
||||
run: |
|
||||
@@ -87,7 +71,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [integration-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
@@ -96,13 +80,13 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Integration Container Environment
|
||||
run: |
|
||||
@@ -134,7 +118,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [e2e-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 2.0 --memory 2g --hostname kafka-e2e-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
@@ -143,12 +127,12 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -313,7 +297,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [consumer-group-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -322,12 +306,12 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -475,7 +459,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [client-compat-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -484,12 +468,12 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -633,7 +617,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [smq-integration-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 2g --hostname kafka-smq-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -642,12 +626,12 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -794,7 +778,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [protocol-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 1g --tmpfs /tmp:exec --hostname kafka-protocol-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -803,13 +787,13 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Protocol Container Environment
|
||||
run: |
|
||||
|
||||
@@ -37,10 +37,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -82,10 +82,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
name: "Master Cold Start Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'weed/operation/**'
|
||||
- 'test/master_cold_start/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'weed/operation/**'
|
||||
- 'test/master_cold_start/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
master-cold-start-tests:
|
||||
name: Master Cold Start Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run master cold start tests
|
||||
# test/master_cold_start boots a fresh master plus empty volume
|
||||
# servers and requires the very first assign (HTTP and gRPC, no
|
||||
# client retries) to complete a write: the assign that triggers
|
||||
# volume growth must wait for it instead of failing with
|
||||
# "volume growth in progress".
|
||||
run: |
|
||||
export WEED_BINARY=$(go env GOPATH)/bin/weed
|
||||
go test -v -timeout=8m ./test/master_cold_start/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/master_cold_start/cluster.go keeps failing-test dirs created
|
||||
# via os.MkdirTemp("", "seaweedfs_master_cold_start_it_") with each
|
||||
# process log under <baseDir>/logs/.
|
||||
mkdir -p /tmp/master-cold-start-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_master_cold_start_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/master-cold-start-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/master-cold-start-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: master-cold-start-test-logs
|
||||
path: /tmp/master-cold-start-logs/
|
||||
retention-days: 7
|
||||
@@ -8,7 +8,6 @@ on:
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/server/master_grpc_server.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
@@ -19,7 +18,6 @@ on:
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/server/master_grpc_server.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
@@ -42,10 +40,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -1,203 +0,0 @@
|
||||
name: "mount: benchmark"
|
||||
|
||||
# Manual benchmark: native WinFsp mount vs rclone+WebDAV on the same Windows
|
||||
# runner, with a Linux FUSE mount of the same build as a reference. Numbers
|
||||
# from shared runners are noisy; this is for finding factor-of-N gaps, not
|
||||
# regressions of a few percent.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [ 'winfsp-bench**' ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
bench-windows:
|
||||
name: Windows native vs rclone
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install WinFsp and rclone
|
||||
run: choco install winfsp rclone -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
- name: Benchmark both mounts
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Wait-Drive($drive, $what) {
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path "${drive}\") { Write-Host "$what is mounted on $drive"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
throw "$what never appeared on $drive"
|
||||
}
|
||||
|
||||
function Invoke-Bench($dir, $label, $out) {
|
||||
Write-Host "::group::bench $label"
|
||||
& go run ./test/mount_bench -dir $dir -label $label -filer 127.0.0.1:8888 -out $out
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "bench $label failed with exit $code" }
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "mini cluster never came up"
|
||||
}
|
||||
Write-Host "filer on 8888/18888, webdav on 7333"
|
||||
|
||||
# --- native WinFsp mount ---
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput C:\seaweed-mount.log -RedirectStandardError C:\seaweed-mount.err.log
|
||||
Wait-Drive 'S:' 'weed mount'
|
||||
|
||||
Invoke-Bench 'S:\bench-native' 'winfsp-native' 'C:\results-native.json'
|
||||
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
$deadline = (Get-Date).AddMinutes(1)
|
||||
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
|
||||
|
||||
# --- rclone + WebDAV on the same WinFsp ---
|
||||
# rclone serves listings from a directory cache it fills lazily, so the
|
||||
# big-listing files have to exist before it mounts or it never sees them.
|
||||
& go run ./test/mount_bench -filer 127.0.0.1:8888 -seed bench-rclone/biglist
|
||||
if ($LASTEXITCODE -ne 0) { throw "seeding failed" }
|
||||
|
||||
$env:RCLONE_CONFIG_SEAWEED_TYPE = 'webdav'
|
||||
$env:RCLONE_CONFIG_SEAWEED_URL = 'http://127.0.0.1:7333'
|
||||
$env:RCLONE_CONFIG_SEAWEED_VENDOR = 'other'
|
||||
Start-Process -FilePath rclone `
|
||||
-ArgumentList 'mount','seaweed:','T:','--vfs-cache-mode=writes','-v','--log-file=C:\rclone.log'
|
||||
Wait-Drive 'T:' 'rclone mount'
|
||||
|
||||
Invoke-Bench 'T:\bench-rclone' 'rclone-webdav' 'C:\results-rclone.json'
|
||||
|
||||
Stop-Process -Name rclone -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# --- comparison ---
|
||||
$table = & go run ./test/mount_bench -compare C:\results-native.json,C:\results-rclone.json
|
||||
$table | Write-Host
|
||||
"## Windows: native WinFsp vs rclone+WebDAV" | Out-File -Append $env:GITHUB_STEP_SUMMARY
|
||||
$table | Out-File -Append $env:GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\rclone.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 100 }
|
||||
}
|
||||
|
||||
- name: Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: results-windows
|
||||
path: C:\results-*.json
|
||||
if-no-files-found: ignore
|
||||
|
||||
bench-linux:
|
||||
name: Linux FUSE reference
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Allow non-root FUSE mounts with allow_other
|
||||
run: |
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o /tmp/weed ./weed
|
||||
|
||||
- name: Benchmark FUSE mount
|
||||
run: |
|
||||
set -e
|
||||
mkdir -p /tmp/seaweed-data
|
||||
/tmp/weed -logtostderr mini -dir=/tmp/seaweed-data -ip=127.0.0.1 > /tmp/mini.log 2>&1 &
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if nc -z 127.0.0.1 8888 && nc -z 127.0.0.1 18888; then break; fi
|
||||
sleep 3
|
||||
done
|
||||
nc -z 127.0.0.1 8888 || { cat /tmp/mini.log; echo "filer never came up"; exit 1; }
|
||||
|
||||
mkdir -p "$HOME/mnt"
|
||||
/tmp/weed -logtostderr mount -filer=127.0.0.1:8888 -dir="$HOME/mnt" > /tmp/mount.log 2>&1 &
|
||||
for i in $(seq 1 60); do
|
||||
if mountpoint -q "$HOME/mnt"; then break; fi
|
||||
sleep 2
|
||||
done
|
||||
mountpoint -q "$HOME/mnt" || { cat /tmp/mount.log; echo "mount never appeared"; exit 1; }
|
||||
|
||||
go run ./test/mount_bench -dir "$HOME/mnt/bench-linux" -label linux-fuse -filer 127.0.0.1:8888 -out /tmp/results-linux.json
|
||||
|
||||
{
|
||||
echo "## Linux FUSE reference"
|
||||
go run ./test/mount_bench -compare /tmp/results-linux.json
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
run: |
|
||||
tail -n 100 /tmp/mount.log /tmp/mini.log 2>/dev/null || true
|
||||
|
||||
- name: Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: results-linux
|
||||
path: /tmp/results-linux.json
|
||||
if-no-files-found: ignore
|
||||
@@ -1,124 +0,0 @@
|
||||
name: "mount: windows conformance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
conformance:
|
||||
name: WinFsp conformance
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and run winfsp-tests
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Write-Host "::group::winfsp-tests"
|
||||
& pwsh -File test/winfsp-conformance/run.ps1 -MountPoint S:\
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "winfsp-tests failed with exit $code" }
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -1,194 +0,0 @@
|
||||
name: "mount: windows"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
mount-windows:
|
||||
name: Mount on Windows
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and exercise
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
function Stop-Mount {
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
$deadline = (Get-Date).AddMinutes(1)
|
||||
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
|
||||
if (Test-Path S:\) { throw "S: still present after stopping the mount" }
|
||||
Write-Host "unmounted"
|
||||
}
|
||||
|
||||
function Invoke-Tests($label, [string[]]$goArgs) {
|
||||
Write-Host "::group::$label"
|
||||
& go @goArgs
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "$label failed with exit $code" }
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Invoke-Tests 'exercise' @('test','-v','-timeout','20m','./test/winfsp','-mountpoint=S:\')
|
||||
Invoke-Tests 'persist-write' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=write','-filer=127.0.0.1:8888')
|
||||
|
||||
Stop-Mount
|
||||
Start-Mount 'seaweed-remount'
|
||||
|
||||
Invoke-Tests 'persist-verify' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=verify')
|
||||
|
||||
# WinFsp creates the mount directory itself, so the path must not
|
||||
# exist; only its parent has to.
|
||||
Write-Host "::group::mount over a directory"
|
||||
Stop-Mount
|
||||
Remove-Item C:\seaweed-mnt -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=C:\seaweed-mnt' `
|
||||
-RedirectStandardOutput C:\seaweed-dirmount.log -RedirectStandardError C:\seaweed-dirmount.err.log
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
$ok = $false
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# Listing succeeds on the plain empty directory too, so wait for the
|
||||
# reparse point WinFsp turns it into. Otherwise this step passes
|
||||
# without a mount and writes to local disk.
|
||||
$item = Get-Item C:\seaweed-mnt -Force -ErrorAction SilentlyContinue
|
||||
if ($null -ne $item -and $item.Attributes.ToString() -like '*ReparsePoint*') { $ok = $true; break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
if (-not $ok) {
|
||||
Get-Content C:\seaweed-dirmount.log, C:\seaweed-dirmount.err.log -ErrorAction SilentlyContinue
|
||||
throw "mounting over a directory failed"
|
||||
}
|
||||
Set-Content -Path C:\seaweed-mnt\dirmount.txt -Value 'via directory mount'
|
||||
if ((Get-Content C:\seaweed-mnt\dirmount.txt) -ne 'via directory mount') { throw "readback through the directory mount differs" }
|
||||
Remove-Item C:\seaweed-mnt\dirmount.txt -Force
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
Start-Sleep -Seconds 5
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
Start-Mount 'seaweed-remount2'
|
||||
|
||||
Write-Host "::group::explorer-style walk"
|
||||
New-Item -ItemType Directory -Force -Path S:\walk | Out-Null
|
||||
1..200 | ForEach-Object { Set-Content -Path "S:\walk\f$_.txt" -Value "line $_" }
|
||||
$names = @(Get-ChildItem S:\walk | ForEach-Object { $_.Name })
|
||||
if ($names.Count -ne 200) {
|
||||
# Name the strays: a dot entry surfacing here is a different problem
|
||||
# from a missing or duplicated file.
|
||||
$unexpected = $names | Where-Object { $_ -notmatch '^f\d+\.txt$' }
|
||||
throw "listed $($names.Count) entries, expected 200; unexpected: $($unexpected -join ', ')"
|
||||
}
|
||||
$body = Get-Content S:\walk\f42.txt
|
||||
if ($body -ne 'line 42') { throw "unexpected content: $body" }
|
||||
Copy-Item S:\walk\f42.txt S:\walk\copy.txt
|
||||
Remove-Item S:\walk -Recurse -Force
|
||||
if (Test-Path S:\walk) { throw "directory survived recursive delete" }
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-remount.log','C:\seaweed-remount.err.log','C:\seaweed-remount2.log','C:\seaweed-remount2.err.log','C:\seaweed-dirmount.log','C:\seaweed-dirmount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -34,10 +34,10 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
name: "NFS Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/nfs-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
nfs-integration:
|
||||
name: NFS Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run NFS Integration Tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running NFS integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run the protocol-layer tests. The kernel-mount tests require root
|
||||
# for mount(2) and are exercised in their own privileged step below;
|
||||
# skip them here so a "skipped because not root" line doesn't show
|
||||
# up as noise on every CI run.
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "NFS integration tests completed"
|
||||
|
||||
- name: Install kernel NFS client
|
||||
run: |
|
||||
# nfs-common provides mount.nfs; netbase provides /etc/protocols
|
||||
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nfs-common netbase
|
||||
|
||||
- name: Run kernel-mount E2E tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running kernel-mount end-to-end tests..."
|
||||
echo "These mount the running 'weed nfs' subprocess via the actual"
|
||||
echo "Linux NFS client to catch protocol regressions invisible to"
|
||||
echo "the go-nfs-client-based tests above."
|
||||
echo "============================================"
|
||||
|
||||
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
|
||||
# binary that test/nfs/framework.go locates via $PATH) resolve
|
||||
# correctly under sudo, and pass through the Go module/cache dirs
|
||||
# so we don't redownload modules under root.
|
||||
sudo env "PATH=$PATH" \
|
||||
GOMODCACHE="$(go env GOMODCACHE)" \
|
||||
GOCACHE="$(go env GOCACHE)" \
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Kernel-mount E2E tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
|
||||
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
|
||||
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
|
||||
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
|
||||
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1,469 +0,0 @@
|
||||
name: "Performance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- 'seaweed-volume/**'
|
||||
- 'test/perf/**'
|
||||
- '.github/workflows/performance.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
profile_duration:
|
||||
description: "CPU profiling duration in seconds"
|
||||
required: false
|
||||
default: "30"
|
||||
type: string
|
||||
benchmark_files:
|
||||
description: "Number of files for the throughput benchmark"
|
||||
required: false
|
||||
default: "100000"
|
||||
type: string
|
||||
benchmark_concurrency:
|
||||
description: "Concurrent read/write workers"
|
||||
required: false
|
||||
default: "16"
|
||||
type: string
|
||||
benchmark_size:
|
||||
description: "Simulated file size in bytes"
|
||||
required: false
|
||||
default: "1024"
|
||||
type: string
|
||||
s3_objects:
|
||||
description: "Number of objects for the S3 benchmark"
|
||||
required: false
|
||||
default: "20000"
|
||||
type: string
|
||||
s3_size:
|
||||
description: "S3 object size in bytes"
|
||||
required: false
|
||||
default: "4096"
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/performance
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
VOL_SIZE_LIMIT: "1024"
|
||||
|
||||
jobs:
|
||||
performance-profile:
|
||||
name: CPU and Heap Profile
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Start server with profiling enabled
|
||||
run: |
|
||||
mkdir -p ./perfdata
|
||||
./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \
|
||||
-s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \
|
||||
-s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> weed.log 2>&1 &
|
||||
echo "WEED_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# give the volume server a moment to register with the master
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Capture profiles under load
|
||||
run: |
|
||||
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
|
||||
# drive write load so the sampled profile reflects real work
|
||||
./weed_bin benchmark -master=localhost:9333 -writeOnly \
|
||||
-c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 &
|
||||
echo "Sampling CPU profile for ${DURATION}s..."
|
||||
curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt
|
||||
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
|
||||
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS)"
|
||||
echo '```'
|
||||
if [ -f mem-profile.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Profile summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## CPU profile (top functions)"
|
||||
echo '```'
|
||||
head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop server
|
||||
if: always()
|
||||
run: kill "${WEED_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: tail -200 weed.log || true
|
||||
|
||||
- name: Upload profiles
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-profile-${{ github.run_number }}
|
||||
path: |
|
||||
cpu.pprof
|
||||
heap.pprof
|
||||
cpu-top.txt
|
||||
heap-top.txt
|
||||
goroutine.txt
|
||||
mem-profile.csv
|
||||
mem-profile.csv.peak
|
||||
weed.log
|
||||
retention-days: 30
|
||||
|
||||
benchmark:
|
||||
name: Throughput Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start master and volume server
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# let the volume server register with the master via heartbeat
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-benchmark.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run throughput benchmark
|
||||
run: |
|
||||
N="${{ github.event.inputs.benchmark_files || '100000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.benchmark_size || '1024' }}"
|
||||
./weed_bin benchmark -master=localhost:9333 \
|
||||
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
|
||||
|
||||
- name: Run Go micro-benchmarks
|
||||
if: matrix.impl == 'go'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
go test -run='^$' -bench=. -benchmem -benchtime=10x \
|
||||
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
|
||||
2>&1 | tee go-benchmarks.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-benchmark.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Throughput benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
|
||||
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
|
||||
- name: Upload benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
benchmark-results.txt
|
||||
go-benchmarks.txt
|
||||
mem-benchmark.csv
|
||||
mem-benchmark.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
retention-days: 7
|
||||
|
||||
s3-benchmark:
|
||||
name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed and S3 load tool
|
||||
run: |
|
||||
go build -o weed_bin ./weed
|
||||
go build -o s3bench ./test/s3/benchmark
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start cluster with S3 gateway
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 3
|
||||
./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
|
||||
-s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> filer.log 2>&1 &
|
||||
echo "FILER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z localhost 8000 2>/dev/null; then
|
||||
echo "s3 gateway is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 2
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-s3.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run S3 read/write benchmark
|
||||
run: |
|
||||
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.s3_size || '4096' }}"
|
||||
./s3bench -endpoint=http://localhost:8000 \
|
||||
-access-key=some_access_key1 -secret-key=some_secret_key1 \
|
||||
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
|
||||
2>&1 | tee s3-benchmark-results.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-s3.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: S3 benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## S3 read/write benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
|
||||
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
|
||||
|
||||
- name: Upload S3 benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
s3-benchmark-results.txt
|
||||
mem-s3.csv
|
||||
mem-s3.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
filer.log
|
||||
retention-days: 7
|
||||
@@ -32,10 +32,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "Plugin Worker Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -39,13 +27,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Run plugin worker tests
|
||||
run: go test -v ./${{ matrix.path }}
|
||||
|
||||
@@ -3,24 +3,8 @@ name: "PostgreSQL Gateway Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/postgres-tests
|
||||
@@ -39,24 +23,19 @@ jobs:
|
||||
working-directory: test/postgres
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
name: "release: bump version and cut the release"
|
||||
|
||||
# One entry point for a SeaweedFS release:
|
||||
# 1. bump MAJOR/MINOR in constants.go and the Helm Chart.yaml, commit to master
|
||||
# 2. push the <appVersion> tag, which fans out to the workflows that trigger on
|
||||
# `push: tags` (binaries_release*, container_release_unified, helm_manual_release)
|
||||
# 3. create the GitHub release, with GitHub's generated notes
|
||||
# 4. dispatch "Prepare release" in seaweedfs-csi-driver and seaweedfs-operator,
|
||||
# which pick up the new master through `go get -u`, and wait for both
|
||||
#
|
||||
# Events raised by the default GITHUB_TOKEN do not start other workflows, and it
|
||||
# cannot reach the other two repositories at all. Add a repo secret RELEASE_PAT
|
||||
# with `contents: write` here and `actions: write` on the csi-driver and operator
|
||||
# repos. Without it the tag is still pushed, but nothing downstream of it runs.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
bump:
|
||||
description: "Which part to increment (ignored when 'version' is set)"
|
||||
type: choice
|
||||
options:
|
||||
- minor
|
||||
- major
|
||||
default: minor
|
||||
version:
|
||||
description: "Explicit MAJOR.MINOR to set, e.g. 4.36 (overrides 'bump')"
|
||||
type: string
|
||||
required: false
|
||||
downstream:
|
||||
description: "Also release the CSI driver and the operator"
|
||||
type: boolean
|
||||
default: true
|
||||
dry_run:
|
||||
description: "Show the version bump, but change nothing"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
outputs:
|
||||
app_version: ${{ steps.compute.outputs.app_version }}
|
||||
sha: ${{ steps.tag.outputs.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: master
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Check the release token
|
||||
env:
|
||||
HAS_PAT: ${{ secrets.RELEASE_PAT != '' }}
|
||||
run: |
|
||||
if [ "$HAS_PAT" != "true" ]; then
|
||||
echo "::warning::RELEASE_PAT is not set. The tag will be pushed with GITHUB_TOKEN, so the binary, container and helm workflows will not start on their own."
|
||||
fi
|
||||
|
||||
- name: Compute new version
|
||||
id: compute
|
||||
env:
|
||||
BUMP: ${{ inputs.bump }}
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
|
||||
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
echo "current: ${MAJOR}.${MINOR}"
|
||||
|
||||
if [ -n "$INPUT_VERSION" ]; then
|
||||
if ! [[ "$INPUT_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR, e.g. 5.01 (got '$INPUT_VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
# 10# forces base 10 so 08/09 are not parsed as octal.
|
||||
MAJOR=$((10#${INPUT_VERSION%%.*}))
|
||||
MINOR=$((10#${INPUT_VERSION##*.}))
|
||||
if [ "$MINOR" -gt 99 ]; then
|
||||
echo "::error::minor must be 0-99 (got $MINOR); it rolls into the next major at 99"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
case "$BUMP" in
|
||||
# Minor is a 2-digit field: 4.99 -> 5.00 -> 5.01.
|
||||
major) MAJOR=$((MAJOR + 1)); MINOR=0 ;;
|
||||
minor)
|
||||
if [ "$MINOR" -ge 99 ]; then
|
||||
MAJOR=$((MAJOR + 1)); MINOR=0
|
||||
else
|
||||
MINOR=$((MINOR + 1))
|
||||
fi
|
||||
;;
|
||||
*) echo "::error::unknown bump '$BUMP'"; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# appVersion mirrors the Go VERSION_NUMBER (zero-padded minor);
|
||||
# chart version is plain SemVer (no leading zeros).
|
||||
APP_VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
|
||||
CHART_VERSION="${MAJOR}.${MINOR}.0"
|
||||
echo "new: app=${APP_VERSION} chart=${CHART_VERSION}"
|
||||
|
||||
{
|
||||
echo "major=${MAJOR}"
|
||||
echo "minor=${MINOR}"
|
||||
echo "app_version=${APP_VERSION}"
|
||||
echo "chart_version=${CHART_VERSION}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Apply version to constants.go
|
||||
env:
|
||||
MAJOR: ${{ steps.compute.outputs.major }}
|
||||
MINOR: ${{ steps.compute.outputs.minor }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
sed -i -E "s/(MAJOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MAJOR}\2/" "$CONST"
|
||||
sed -i -E "s/(MINOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MINOR}\2/" "$CONST"
|
||||
grep -E 'MAJOR_VERSION|MINOR_VERSION' "$CONST"
|
||||
|
||||
- name: Apply version to Chart.yaml
|
||||
env:
|
||||
APP_VERSION: ${{ steps.compute.outputs.app_version }}
|
||||
CHART_VERSION: ${{ steps.compute.outputs.chart_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHART=k8s/charts/seaweedfs/Chart.yaml
|
||||
sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "$CHART"
|
||||
sed -i -E "s/^version:.*/version: ${CHART_VERSION}/" "$CHART"
|
||||
cat "$CHART"
|
||||
|
||||
- name: Commit, and push the tag
|
||||
id: tag
|
||||
env:
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then
|
||||
echo "::error::Tag ${TAG} already exists."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
git --no-pager diff --stat
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
if git diff --quiet; then
|
||||
echo "::warning::Version files are already at ${TAG}; tagging the current HEAD."
|
||||
else
|
||||
git add weed/util/version/constants.go k8s/charts/seaweedfs/Chart.yaml
|
||||
git commit -m "${TAG}"
|
||||
git push
|
||||
fi
|
||||
|
||||
# Push the tag with git so the `push: tags` triggers fire. Creating the
|
||||
# tag through the release API alone would only emit a `create` event.
|
||||
git tag "$TAG"
|
||||
git push origin "$TAG"
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create the release
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: gh release create "$TAG" --title "$TAG" --generate-notes --verify-tag
|
||||
|
||||
downstream:
|
||||
needs: release
|
||||
if: ${{ inputs.downstream && !inputs.dry_run }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- repo: seaweedfs/seaweedfs-csi-driver
|
||||
workflow: prepare_release.yaml
|
||||
- repo: seaweedfs/seaweedfs-operator
|
||||
workflow: prepare_release.yml
|
||||
steps:
|
||||
- name: Release ${{ matrix.repo }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT }}
|
||||
REPO: ${{ matrix.repo }}
|
||||
WORKFLOW: ${{ matrix.workflow }}
|
||||
SHA: ${{ needs.release.outputs.sha }}
|
||||
MODULE: github.com/seaweedfs/seaweedfs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN}" ]; then
|
||||
echo "::error::RELEASE_PAT with actions:write on ${REPO} is required to release it"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The dispatched workflow pins seaweedfs with `go get -u ...@latest`, so
|
||||
# wait until the proxy serves the release commit as the tip. Asking for
|
||||
# the commit by name is what makes the proxy fetch it.
|
||||
for _ in $(seq 30); do
|
||||
curl -sf "https://proxy.golang.org/${MODULE}/@v/${SHA}.info" >/dev/null || true
|
||||
TIP=$(curl -sf "https://proxy.golang.org/${MODULE}/@latest" | jq -r '.Origin.Hash // ""' || true)
|
||||
[ "$TIP" = "$SHA" ] && break
|
||||
sleep 10
|
||||
done
|
||||
if [ "$TIP" != "$SHA" ]; then
|
||||
echo "::error::the module proxy still serves ${TIP} as the tip, so ${REPO} would pin a pre-release commit. Run ${WORKFLOW} there once it catches up."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait on the release the dispatched workflow publishes, not on the run
|
||||
# that publishes it: a dispatch cannot be told apart from a concurrent
|
||||
# one through the API, and the release is what we are here for.
|
||||
released() { gh api "repos/${REPO}/releases?per_page=30" --jq '[.[].tag_name]'; }
|
||||
|
||||
BEFORE=$(released)
|
||||
gh workflow run -R "$REPO" "$WORKFLOW" --ref master -f bump=patch -f update_seaweedfs=true
|
||||
|
||||
for _ in $(seq 80); do
|
||||
sleep 15
|
||||
NEW=$(released | jq -c --argjson before "$BEFORE" '. - $before')
|
||||
[ "$(jq length <<<"$NEW")" -gt 0 ] && break
|
||||
done
|
||||
if [ "$(jq length <<<"$NEW")" -eq 0 ]; then
|
||||
echo "::error::${REPO} published no release within 20 minutes; see https://github.com/${REPO}/actions/workflows/${WORKFLOW}"
|
||||
exit 1
|
||||
fi
|
||||
echo "${REPO} released $(jq -r 'join(", ")' <<<"$NEW")"
|
||||
@@ -34,27 +34,24 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
rust-
|
||||
|
||||
- name: Build Rust volume server
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
@@ -69,32 +66,29 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
@@ -140,9 +134,6 @@ jobs:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -151,32 +142,29 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
@@ -188,28 +176,30 @@ jobs:
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests with Rust volume
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
run: |
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
@@ -230,6 +220,23 @@ jobs:
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
name: "Rust Plugin Worker Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-worker/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-worker/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
rust-worker-build:
|
||||
name: Rust Plugin Worker Build and Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-worker/target/release
|
||||
key: rust-worker-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-worker/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-worker-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
protoc=$(find ~/.cargo/registry/src -path '*protoc-bin-vendored-linux-x86_64*/bin/protoc' | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
# The release profile is what ships, and it is where the release and the
|
||||
# container builds would otherwise discover a break for the first time.
|
||||
- name: Build the plugin workers
|
||||
run: cd seaweed-worker && cargo build --release
|
||||
|
||||
# The tests that need a live gateway skip themselves without one, the way
|
||||
# the Go integration tests skip without Docker; the lifecycle suite in
|
||||
# test/s3tables/lifecycle is what runs them against a real cluster.
|
||||
# Release, so this reuses the build above rather than compiling lance,
|
||||
# arrow and datafusion a second time in another profile.
|
||||
- name: Run unit tests
|
||||
run: cd seaweed-worker && cargo test --release --workspace
|
||||
@@ -39,13 +39,16 @@ jobs:
|
||||
asset_suffix: linux-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -105,7 +108,10 @@ jobs:
|
||||
asset_suffix: darwin-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -113,7 +119,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: "rust: build versioned binaries"
|
||||
name: "rust: build versioned volume server binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -26,7 +26,10 @@ jobs:
|
||||
cross: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -48,7 +51,7 @@ jobs:
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -63,42 +66,34 @@ jobs:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
# Large disk (default, 5bytes feature)
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
# Normal volume size
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -109,102 +104,7 @@ jobs:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
# The Rust maintenance worker: Linux only, because it runs beside the cluster
|
||||
# it maintains rather than on a laptop, and its dependency tree (lance, arrow,
|
||||
# datafusion) makes every extra target an expensive build.
|
||||
build-rust-worker-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
asset_suffix: linux_arm64
|
||||
cross: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# The upload step is handed a token explicitly; a cargo build script
|
||||
# should not find another one sitting in the checkout's git config.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-worker-release-${{ matrix.target }}-${{ hashFiles('seaweed-worker/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-worker-release-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
protoc=$(find ~/.cargo/registry/src -path '*protoc-bin-vendored-linux-x86_64*/bin/protoc' | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
|
||||
- name: Package binary
|
||||
run: |
|
||||
cp seaweed-worker/target/${{ matrix.target }}/release/weed-worker weed-worker
|
||||
tar czf weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker
|
||||
rm weed-worker
|
||||
md5sum weed-worker_${{ matrix.asset_suffix }}.tar.gz > weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-worker-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
@@ -219,7 +119,10 @@ jobs:
|
||||
asset_suffix: darwin_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -227,7 +130,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -242,40 +145,32 @@ jobs:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5 -r "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -286,9 +181,7 @@ jobs:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
@@ -296,13 +189,16 @@ jobs:
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: choco install protoc -y
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -317,42 +213,33 @@ jobs:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target-dir target/large-disk
|
||||
cargo build --release
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --no-default-features --target-dir target/normal
|
||||
cargo build --release --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
shell: bash
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
|
||||
rm weed-volume-large-disk.exe
|
||||
|
||||
cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
|
||||
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Generate md5 checksums
|
||||
shell: bash
|
||||
run: |
|
||||
for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -363,6 +250,4 @@ jobs:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
|
||||
@@ -34,10 +34,10 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -2,16 +2,7 @@ name: "S3 Authenticated Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/normal/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-example-integration-tests.yml'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-integration-tests
|
||||
cancel-in-progress: true
|
||||
@@ -27,10 +18,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -2,15 +2,7 @@ name: "S3 Filer Group Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/filer_group/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-filer-group-tests.yml'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
@@ -30,10 +22,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -2,15 +2,7 @@ name: "S3 Go Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-go-tests.yml'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-go-tests
|
||||
cancel-in-progress: true
|
||||
@@ -33,10 +25,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -97,10 +89,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -145,10 +137,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -196,10 +188,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -286,10 +278,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -331,10 +323,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -389,10 +381,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -453,10 +445,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -502,10 +494,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -570,10 +562,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -625,10 +617,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -39,10 +39,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -88,10 +88,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -202,10 +202,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -262,10 +262,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -35,10 +35,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -26,10 +26,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
@@ -38,22 +38,7 @@ jobs:
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
# Run every versioning test, so a regression test lands covered instead
|
||||
# of waiting for someone to remember this file. Name a test in EXCLUDE,
|
||||
# with the reason, to keep it out.
|
||||
#
|
||||
# TestVersioningPagination*: opt-in stress tests that build 1500+
|
||||
# versions. They self-skip without ENABLE_STRESS_TESTS and have their
|
||||
# own make target, so this gate should not carry them.
|
||||
EXCLUDE='TestVersioningPagination.*'
|
||||
tests=$(go test . -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and pass this job vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; }
|
||||
selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$")
|
||||
[ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; }
|
||||
echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests"
|
||||
# make swallows a lone trailing $, taking the anchor with it, so escape it.
|
||||
make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$'
|
||||
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
@@ -79,10 +64,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
@@ -117,10 +102,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -36,16 +36,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
@@ -143,12 +143,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
|
||||
- name: Run Go unit tests
|
||||
|
||||
@@ -43,10 +43,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -86,10 +86,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -195,10 +195,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -313,10 +313,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -400,10 +400,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -3,22 +3,8 @@ name: "S3 Proxy Signature Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
|
||||
@@ -34,19 +20,14 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
|
||||
@@ -28,10 +28,10 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -25,27 +25,23 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
run: docker pull apache/spark:3.5.8
|
||||
|
||||
- name: Run S3 Spark integration tests
|
||||
working-directory: test/s3/spark
|
||||
|
||||
@@ -44,10 +44,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -112,10 +112,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -160,10 +160,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -209,10 +209,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -258,10 +258,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -309,10 +309,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -357,10 +357,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -2,15 +2,6 @@ name: "S3 Tables Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/worker/tasks/iceberg/**'
|
||||
- 'test/s3tables/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-tables-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -24,10 +15,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -84,19 +75,14 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
@@ -145,23 +131,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Trino image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull trinodb/trino:479
|
||||
run: docker pull trinodb/trino:479
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -215,24 +197,22 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull dremio/dremio-oss:25.2.0
|
||||
pull python:3.11-slim
|
||||
- name: Pre-pull Dremio image
|
||||
run: docker pull dremio/dremio-oss:25.2.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -288,24 +268,22 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/doris:doris-all-in-one-2.1.0
|
||||
pull python:3.11-slim
|
||||
- name: Pre-pull Doris image
|
||||
run: docker pull apache/doris:doris-all-in-one-2.1.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -354,91 +332,6 @@ jobs:
|
||||
path: test/s3tables/catalog_doris/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
clickhouse-iceberg-catalog-tests:
|
||||
name: ClickHouse Iceberg Catalog Integration Tests (${{ matrix.tag }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Pinned baseline, and latest so new ClickHouse releases are
|
||||
# exercised without a code change.
|
||||
- clickhouse-image: clickhouse/clickhouse-server:25.8
|
||||
tag: "25.8"
|
||||
- clickhouse-image: clickhouse/clickhouse-server:latest
|
||||
tag: latest
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull ${{ matrix.clickhouse-image }}
|
||||
pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run ClickHouse Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_clickhouse
|
||||
env:
|
||||
CLICKHOUSE_IMAGE: ${{ matrix.clickhouse-image }}
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting ClickHouse Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "ClickHouse Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_clickhouse
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|clickhouse)" || true
|
||||
echo "=== ClickHouse containers ==="
|
||||
docker ps -a --filter "name=seaweed-clickhouse" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: clickhouse-iceberg-catalog-test-logs-${{ matrix.tag }}
|
||||
path: test/s3tables/catalog_clickhouse/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -446,10 +339,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -461,15 +354,8 @@ jobs:
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Pre-pull Polaris image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/polaris:latest
|
||||
run: docker pull apache/polaris:latest
|
||||
|
||||
- name: Run Polaris Integration Tests
|
||||
timeout-minutes: 25
|
||||
@@ -514,23 +400,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
run: docker pull apache/spark:3.5.1
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -584,24 +466,21 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull RisingWave image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull risingwavelabs/risingwave:v2.5.0
|
||||
pull postgres:16-alpine
|
||||
docker pull risingwavelabs/risingwave:v2.5.0
|
||||
docker pull postgres:16-alpine
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -655,47 +534,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Week stamp for image cache key
|
||||
id: week
|
||||
run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore python:3 image cache
|
||||
id: python-image
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/python3-image.tar
|
||||
key: python3-image-${{ steps.week.outputs.week }}
|
||||
restore-keys: |
|
||||
python3-image-
|
||||
|
||||
- name: Load or pull python:3
|
||||
run: |
|
||||
if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then
|
||||
docker load -i /tmp/python3-image.tar
|
||||
exit 0
|
||||
fi
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
if pull python:3; then
|
||||
docker save -o /tmp/python3-image.tar python:3
|
||||
elif [ -f /tmp/python3-image.tar ]; then
|
||||
# Docker Hub unreachable; fall back to last week's cached image
|
||||
docker load -i /tmp/python3-image.tar
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -749,14 +600,23 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
|
||||
- name: Pre-pull LocalStack image (if needed)
|
||||
run: docker pull localstack/localstack:latest || true
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
@@ -809,24 +669,22 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull unitycatalog/unitycatalog:v0.4.0
|
||||
pull python:3.11-slim
|
||||
- name: Pre-pull Unity Catalog image
|
||||
run: docker pull unitycatalog/unitycatalog:v0.4.0
|
||||
|
||||
- name: Pre-pull Python image for delta-rs writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
@@ -875,293 +733,6 @@ jobs:
|
||||
path: test/s3tables/unity_catalog/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
lancedb-namespace-tests:
|
||||
name: LanceDB Namespace Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed && go build -buildvcs=false .
|
||||
|
||||
- name: Run LanceDB Namespace Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_lancedb
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting LanceDB Namespace Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "LanceDB namespace integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_lancedb
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: lancedb-namespace-test-logs
|
||||
path: test/s3tables/catalog_lancedb/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
table-lifecycle-tests:
|
||||
name: Table Lifecycle Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull python:3.11-slim
|
||||
pull duckdb/duckdb:latest
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed && go build -buildvcs=false .
|
||||
|
||||
- name: Run Table Lifecycle Integration Tests
|
||||
timeout-minutes: 35
|
||||
working-directory: test/s3tables/lifecycle
|
||||
env:
|
||||
# The Rust worker's own tests cover its handlers; a cold build of the
|
||||
# lance crate costs more here than the layer it would be checking.
|
||||
WEED_LANCE_MAINTENANCE: library
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
|
||||
echo "Table lifecycle integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/lifecycle
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: table-lifecycle-test-logs
|
||||
path: test/s3tables/lifecycle/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
duckdb-lance-tests:
|
||||
name: DuckDB Lance Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
# The job uploads a test log on failure; nothing here needs to push,
|
||||
# so do not leave a token in the checkout for it to pick up.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull duckdb/duckdb:latest
|
||||
pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed && go build -buildvcs=false .
|
||||
|
||||
- name: Run DuckDB Lance Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_duckdb_lance
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting DuckDB Lance Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "DuckDB Lance integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_duckdb_lance
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|duckdb)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: duckdb-lance-test-logs
|
||||
path: test/s3tables/catalog_duckdb_lance/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
spark-lance-namespace-tests:
|
||||
name: Spark Lance Namespace Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
# The job uploads a test log on failure; nothing here needs to push,
|
||||
# so do not leave a token in the checkout for it to pick up.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Pre-pull images
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed && go build -buildvcs=false .
|
||||
|
||||
- name: Run Spark Lance Namespace Integration Tests
|
||||
timeout-minutes: 35
|
||||
working-directory: test/s3tables/catalog_spark_lance
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Spark Lance Namespace Tests ==="
|
||||
|
||||
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
|
||||
echo "Spark Lance namespace integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_spark_lance
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|spark)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: spark-lance-namespace-test-logs
|
||||
path: test/s3tables/catalog_spark_lance/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -1169,10 +740,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -1234,10 +805,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -1285,10 +856,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -3,26 +3,8 @@ name: "Ceph S3 tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/compatibility/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/compatibility/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3tests
|
||||
@@ -38,16 +20,16 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
@@ -63,7 +45,7 @@ jobs:
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
||||
python3 test/s3/fix_s3_tests_bucket_conflicts.py
|
||||
env:
|
||||
S3_TESTS_PATH: s3-tests
|
||||
|
||||
@@ -381,16 +363,16 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
@@ -406,7 +388,7 @@ jobs:
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
||||
python3 test/s3/fix_s3_tests_bucket_conflicts.py
|
||||
env:
|
||||
S3_TESTS_PATH: s3-tests
|
||||
|
||||
@@ -551,16 +533,16 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
@@ -676,10 +658,10 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -694,12 +676,10 @@ jobs:
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-copy-test-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
set -x
|
||||
# Each test bucket is its own collection and grows 7 volumes; the suite runs
|
||||
# faster than the heartbeat that returns slots from the deleted collections.
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=300 -volume.preStopSeconds=1 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
|
||||
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
|
||||
pid=$!
|
||||
@@ -767,7 +747,7 @@ jobs:
|
||||
sleep 2
|
||||
done
|
||||
|
||||
MASTER_ENDPOINT="http://127.0.0.1:9336" go test -v
|
||||
go test -v
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
@@ -778,16 +758,16 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
|
||||
@@ -34,10 +34,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -34,10 +34,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
# SETUP & BUILD
|
||||
# ========================================
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v5
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
cache: maven
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
- name: Cache Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
id: cache-spark
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: spark-3.5.0-bin-hadoop3
|
||||
key: spark-3.5.0-hadoop3
|
||||
|
||||
@@ -24,10 +24,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
name: "terraform: validate and test modules"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: fmt, validate, plan-level tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: fmt check
|
||||
working-directory: terraform
|
||||
run: tofu fmt -recursive -check -diff
|
||||
|
||||
- name: validate core
|
||||
working-directory: terraform/modules/core
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: validate security
|
||||
working-directory: terraform/modules/security
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: plan-level tests (core)
|
||||
working-directory: terraform/modules/core
|
||||
run: tofu test
|
||||
|
||||
- name: validate examples
|
||||
run: |
|
||||
set -e
|
||||
for ex in terraform/examples/*/; do
|
||||
echo "== validate $ex =="
|
||||
tofu -chdir="$ex" init -backend=false -input=false
|
||||
tofu -chdir="$ex" validate
|
||||
done
|
||||
|
||||
smoke:
|
||||
name: local cluster smoke test (real weed)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o "$RUNNER_TEMP/weed" ./weed
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: Run local cluster harness
|
||||
working-directory: terraform/test/local
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
|
||||
|
||||
- name: Run local mTLS cluster harness
|
||||
working-directory: terraform/test/local-secure
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh
|
||||
@@ -3,20 +3,8 @@ name: "test s3 over https using aws-cli"
|
||||
on:
|
||||
push:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
pull_request:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
@@ -32,11 +20,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "TLS Rotation Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -28,13 +16,13 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
|
||||
@@ -2,13 +2,6 @@ name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/tus/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
@@ -29,10 +22,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
@@ -3,20 +3,8 @@ name: "Vacuum Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -28,13 +16,13 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
|
||||
@@ -29,8 +29,6 @@ permissions:
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '30m'
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
|
||||
jobs:
|
||||
volume-server-integration-tests:
|
||||
@@ -45,10 +43,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
@@ -59,26 +57,28 @@ jobs:
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
@@ -99,6 +99,23 @@ jobs:
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -35,7 +35,6 @@ Your support will be really appreciated by me and other supporters!
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.piknik.com)
|
||||
[](https://www.keepsec.ca)
|
||||
[](https://zyner.org)
|
||||
|
||||
---
|
||||
|
||||
@@ -61,14 +60,14 @@ Table of Contents
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
* [Filer Features](#filer-features)
|
||||
* [Example: Using Seaweed Blob Store](#example-using-seaweed-blob-store)
|
||||
* [Example: Using Seaweed Object Store](#example-using-seaweed-object-store)
|
||||
* [Architecture](#object-store-architecture)
|
||||
* [Compared to Other File Systems](#compared-to-other-file-systems)
|
||||
* [Compared to HDFS](#compared-to-hdfs)
|
||||
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
|
||||
* [Compared to GlusterFS](#compared-to-glusterfs)
|
||||
* [Compared to Ceph](#compared-to-ceph)
|
||||
* [Compared to MinIO, RustFS](#compared-to-minio-rustfs)
|
||||
* [Compared to Minio](#compared-to-minio)
|
||||
* [Dev Plan](#dev-plan)
|
||||
* [Installation Guide](#installation-guide)
|
||||
* [Disk Related Topics](#disk-related-topics)
|
||||
@@ -90,7 +89,7 @@ S3_BUCKET=my-bucket \
|
||||
./weed mini -dir=/data
|
||||
```
|
||||
|
||||
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables buckets, each `name` or `name:FORMAT` where the format is `ICEBERG` (the default) or `LANCE`. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
|
||||
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
|
||||
|
||||
The same command starts everything else too:
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
@@ -426,7 +425,7 @@ If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage
|
||||
|
||||
## SeaweedFS Filer ##
|
||||
|
||||
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory structure is an interface that is implemented in many key-value stores or databases.
|
||||
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory sturcture is an interface that is implemented in many key-value stores or databases.
|
||||
|
||||
The content of a file is mapped to one or many blobs, distributed to multiple volumes on multiple volume servers.
|
||||
|
||||
@@ -466,8 +465,7 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa
|
||||
| GlusterFS | hashing | | FUSE, NFS | | |
|
||||
| Ceph | hashing + rules | | FUSE | Yes | |
|
||||
| MooseFS | in memory | | FUSE | | No |
|
||||
| MinIO | separate meta file per drive for each file | | | Yes | No |
|
||||
| RustFS | separate meta file per drive for each file | | | Yes | No |
|
||||
| MinIO | separate meta file for each file | | | Yes | No |
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
@@ -509,26 +507,24 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mong
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
### Compared to MinIO, RustFS ###
|
||||
### Compared to MinIO ###
|
||||
|
||||
Please note, as Apr 25, 2026 MinIO ceased development. It's strongly discouraged to use that unmaintained software with multiple security bugs. RustFS is a MinIO reimplementation in Rust, Apache 2.0 licensed and still developed, keeping MinIO's storage model down to a byte-compatible on-disk format. So the points below apply to both.
|
||||
Please note, as Apr 25, 2026 MinIO ceased developement. It's strongly discouraged to use that unmaintained software with multiple security bugs.
|
||||
|
||||
MinIO followed AWS S3 closely and was ideal for testing for S3 API. It had good UI, policies, versionings, etc. SeaweedFS is trying to catch up here.
|
||||
|
||||
The metadata are in simple files. Each file write incurs extra writes to the corresponding meta file, on every drive of the erasure set. Changing only tags or retention rewrites that meta file on all of them, so the write amplification does not shrink with object size.
|
||||
MinIO metadata were in simple files. Each file write will incur extra writes to corresponding meta file.
|
||||
|
||||
There is no optimization for lots of small files. The files are simply stored as is to local disks.
|
||||
MinIO did not have optimization for lots of small files. The files were simply stored as is to local disks.
|
||||
Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem.
|
||||
|
||||
Multiple disk IO are needed to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
|
||||
MinIO had multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
|
||||
|
||||
Erasure coding is full-time. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
|
||||
MinIO had full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
|
||||
|
||||
No POSIX-like API support.
|
||||
MinIO did not have POSIX-like API support.
|
||||
|
||||
There are specific requirements on storage layout, which makes it hard to scale out and to maintain. An erasure set must be 2 to 16 drives and must divide the drive list symmetrically, and capacity grows or shrinks a whole pool at a time. In SeaweedFS, just start one volume server pointing to the master. That's all.
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
MinIO had specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
|
||||
|
||||
## Dev Plan ##
|
||||
|
||||
@@ -658,8 +654,7 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
|
||||
## Enterprise ##
|
||||
|
||||
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
|
||||
which has advanced features, including data recovery, self-healing storage,
|
||||
customizable erasure coding, EC vacuum and repair, etc.
|
||||
which has a self-healing storage format with better data protection.
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
|
||||
+15
-69
@@ -1,78 +1,24 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported versions
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Security fixes land in the latest release. Please reproduce against a recent
|
||||
release or `master` before reporting; issues that only reproduce on old,
|
||||
unsupported versions are not eligible for a fix or an advisory.
|
||||
If you find a security issue in SeaweedFS, please report it privately:
|
||||
|
||||
## Reporting a vulnerability
|
||||
- Email: support@seaweedfs.com
|
||||
- Do not open a public GitHub issue
|
||||
|
||||
Report privately through GitHub private vulnerability reporting (the "Report a
|
||||
vulnerability" button under the repository's Security tab). This keeps the
|
||||
report, the fix, and any CVE in one place. Do not open a public issue.
|
||||
Please include:
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce (if possible)
|
||||
- Affected versions
|
||||
|
||||
### What a report must include
|
||||
## Response
|
||||
|
||||
We can only act on reports that show real impact. Please include:
|
||||
- We will respond as soon as possible (usually within 1 business day)
|
||||
- We will investigate and work on a fix
|
||||
- We may coordinate disclosure with you
|
||||
|
||||
- Affected version (a release tag or `master` commit you reproduced on)
|
||||
- The exact deployment and configuration: which components are running
|
||||
(master, volume, filer, S3, admin), which ports are reachable by the
|
||||
attacker, and what authentication is enabled
|
||||
- The attacker's starting position: unauthenticated, a valid S3 user, an admin,
|
||||
or someone with access to the internal cluster network
|
||||
- The trust boundary that is crossed (e.g. an unauthenticated client reading
|
||||
another tenant's data, an S3 user escalating to admin)
|
||||
- A minimal, working reproduction or proof of concept
|
||||
- Expected vs. actual behavior
|
||||
## Notes
|
||||
|
||||
A report without a working reproduction and a clear trust boundary is a
|
||||
hardening suggestion, not a vulnerability. We are glad to receive those, but
|
||||
they are handled on the normal issue tracker, not as security advisories.
|
||||
|
||||
### Automated and AI-assisted reports
|
||||
|
||||
Output from static analysis, dependency scanners, fuzzers, or LLMs is welcome
|
||||
only when you have manually validated it and can supply a working reproduction
|
||||
against a supported version, per the requirements above. Raw tool output,
|
||||
speculative findings, or generated reports without a demonstrated exploit will
|
||||
be closed as hardening suggestions.
|
||||
|
||||
## Trust model
|
||||
|
||||
SeaweedFS is built to run with its cluster components (master, volume servers,
|
||||
and the raw filer API) on a trusted network. Those internal APIs are not an
|
||||
authentication boundary unless you explicitly enable a control (for example
|
||||
volume JWT or filer authentication) and that control is bypassed. Exposing an
|
||||
internal port directly to untrusted clients is a deployment mistake, not a
|
||||
vulnerability in SeaweedFS.
|
||||
|
||||
Reports are in scope when they cross a boundary SeaweedFS is meant to enforce,
|
||||
for example:
|
||||
|
||||
- Unauthenticated access to data or operations that require authentication
|
||||
- One S3 identity reading, writing, or deleting another identity's data
|
||||
- Privilege escalation from a normal S3 user to administrative capability
|
||||
- Bypass of Object Lock / retention where it is configured
|
||||
- Remotely triggered data corruption or loss
|
||||
|
||||
Reports are generally out of scope when they require:
|
||||
|
||||
- Direct access to an internal cluster port that is meant to be private
|
||||
- Full master, filer, or volume server access (already a full compromise)
|
||||
- An insecure example configuration rather than a documented secure setup
|
||||
- Local-only impact on a host the attacker already controls
|
||||
|
||||
## CVE assignment
|
||||
|
||||
When a report is confirmed, we publish an advisory and request the CVE through
|
||||
GitHub. CVEs assigned by third parties without coordinating with us, or for
|
||||
issues that do not cross a boundary described above, may be disputed.
|
||||
|
||||
## Response and disclosure
|
||||
|
||||
- We aim to acknowledge a valid report within a few business days.
|
||||
- We will investigate, work on a fix, and coordinate a disclosure timeline
|
||||
with you.
|
||||
- Please allow time for a fix before any public disclosure.
|
||||
- Please allow time for a fix before public disclosure
|
||||
- If you’re unsure whether something is a security issue, feel free to reach out
|
||||
|
||||
@@ -1,696 +0,0 @@
|
||||
# Lance Catalog for SeaweedFS
|
||||
|
||||
A second catalog surface next to the Iceberg REST catalog, speaking the Lance Namespace
|
||||
REST spec, over the same table buckets and the same filer.
|
||||
|
||||
## Why
|
||||
|
||||
Gravitino 1.1 added a Lance REST service and 1.3 ships it as a standalone server; Lakekeeper
|
||||
added Lance in the same window by a completely different route. That is the useful signal:
|
||||
two unrelated catalogs decided independently that Lance had to be first-class, not a niche.
|
||||
The client side is already there — `lance-spark` (`LanceNamespaceSparkCatalog`
|
||||
with `impl=rest`), `lance-ray`, and the generated Python/Java/Rust clients all talk the same
|
||||
OpenAPI. Implementing the spec means those engines work against SeaweedFS with no
|
||||
SeaweedFS-specific code on the client.
|
||||
|
||||
The second reason is that Gravitino's own documentation names the gap it cannot close:
|
||||
DuckDB, pandas and DataFusion "do not support Lance REST natively yet" and have to fetch a
|
||||
location from the catalog and then open the dataset directly. Gravitino cannot help there,
|
||||
because it does not own the storage. SeaweedFS does. That is the whole design opportunity
|
||||
below.
|
||||
|
||||
## Prior art: three families
|
||||
|
||||
Upstream lists twelve catalog implementations, and they fall into three shapes. Knowing
|
||||
which one we are building matters more than any individual API decision.
|
||||
|
||||
**1. Storage-native, no service.** The Lance Directory Catalog. V1 is a directory listing
|
||||
where every `<name>.lance/` child of a prefix is a table; V2 adds a `__manifest` table —
|
||||
itself a Lance table — holding `object_id`/`object_type`/`location` rows, with nested
|
||||
namespaces, hash-prefixed table directories, and optional managed versioning. No server, no
|
||||
credentials, no governance. This is the floor every other implementation has to beat.
|
||||
|
||||
**2. Protocol-native server.** Someone implements the Lance Namespace REST OpenAPI and
|
||||
clients connect with `impl=rest`. Gravitino is the only one of the twelve that does this,
|
||||
and it is what this design proposes.
|
||||
|
||||
**3. Client-side adapters onto an existing catalog.** Nine of the twelve. The Lance client
|
||||
translates namespace operations into whatever the backing catalog already speaks: Apache
|
||||
Polaris, Unity Catalog, AWS Glue, Hive Metastore v2 and v3, Google BigLake, Dataproc,
|
||||
Microsoft OneLake — and Apache Iceberg REST. Two flavors:
|
||||
|
||||
- Catalogs with a real non-Iceberg table concept mark the format directly. Polaris uses its
|
||||
Generic Table API with `format = lance`; Unity uses an `EXTERNAL` table with
|
||||
`table_type=lance` in properties and the path in `storage_location`; Glue uses
|
||||
`EXTERNAL_TABLE` plus `table_type=lance` in `Parameters`, path in
|
||||
`StorageDescriptor.Location`.
|
||||
- Catalogs with no such concept fake one. The Iceberg REST adapter registers **a regular
|
||||
Iceberg table with a dummy schema — a single nullable string column named `dummy`** —
|
||||
carrying the property `table_type=lance`, and treats the Iceberg table location as the
|
||||
Lance dataset root.
|
||||
|
||||
Every adapter in family 3 lands in the same place: `DeclareTable`/`ListTables`/
|
||||
`DescribeTable`/`DeregisterTable` only, `DropNamespace` in RESTRICT mode only,
|
||||
`load_detailed_metadata=false` only, and `managed_versioning=false`. They are a name-to-
|
||||
location map and nothing more.
|
||||
|
||||
Lakekeeper is the instructive outlier. It has the same generic-table concept Polaris has,
|
||||
but no upstream adapter exists for it — there is no `lance-namespace` reference anywhere in
|
||||
its repository and no page for it in the supported-catalogs list. So Polaris's generic tables
|
||||
are reachable from a stock Lance client and Lakekeeper's are not, despite being the same
|
||||
idea. Shipping the concept is not the same as shipping the integration.
|
||||
|
||||
## Gravitino and Lakekeeper: the two opposite bets
|
||||
|
||||
Both shipped Lance support in the same window and did not build the same thing.
|
||||
|
||||
**Gravitino implements the protocol.** Its `lance/` module serves the Lance Namespace REST
|
||||
spec on its own port (`:9101/lance`), so stock `lance-spark` and `lance-ray` connect with
|
||||
`impl=rest` and no vendor-specific client. The cost is governance: storage credentials are
|
||||
static properties on the catalog (`lance.storage.access_key_id`, `secret_access_key`,
|
||||
`endpoint`, `region`, `allow_http`), optionally overridden per table, handed to the engine
|
||||
as-is. No STS, no expiry, no per-table scoping.
|
||||
|
||||
**Lakekeeper refuses the protocol and governs the object instead.** There is no
|
||||
`lance-namespace` anywhere in the repository; Lance arrived in 0.13.0 (2026-06-30, issue
|
||||
#1673 `Generic Table API with Lance`) as one `format` string on a Lakekeeper-native Generic
|
||||
Table API:
|
||||
|
||||
```
|
||||
POST/GET/DELETE /lakekeeper/v1/{prefix}/namespaces/{ns}/generic-tables[/{table}]
|
||||
GET /lakekeeper/v1/{prefix}/namespaces/{ns}/generic-tables/{table}/credentials
|
||||
POST /lakekeeper/v1/{prefix}/generic-tables/rename
|
||||
```
|
||||
|
||||
`format` is opaque, `schema` and `statistics` are stored but never validated, and the
|
||||
catalog writes no format-specific metadata — engines go straight to the location. In
|
||||
exchange Lance tables get everything Iceberg tables get: STS-vended prefix-scoped
|
||||
credentials, OpenFGA per-action permissions (16 actions), soft-delete with undrop, a
|
||||
protection flag, rename, pagination, and name uniqueness across Iceberg tables, views and
|
||||
generic tables in one namespace. The price is that no stock Lance client can talk to it —
|
||||
you need `pylakekeeper`, which exists mainly to translate vended credentials into
|
||||
`lance_storage_options`.
|
||||
|
||||
So: protocol fidelity and weak governance, or strong governance and client lock-in. Both
|
||||
documented their limit honestly, and it is the same limit. Lakekeeper's capability table
|
||||
says it outright — "Commit coordination: the catalog does not arbitrate writes — engines
|
||||
write directly." Gravitino does not claim it either. Neither of them coordinates a Lance
|
||||
commit, which is exactly the thing a store can do and a control plane cannot.
|
||||
|
||||
We do not have to choose. Serve the Lance protocol natively the way Gravitino does, over
|
||||
the `s3tables` entries that already carry ARNs, policies, tags and maintenance config, and
|
||||
the governance comes from the layer underneath rather than from a proprietary API on top.
|
||||
That is only available to us because we are the store, which is also what makes the third
|
||||
option — arbitrating the commit — available.
|
||||
|
||||
## We are probably already a Lance catalog, and that is a problem
|
||||
|
||||
The Iceberg REST adapter does not care whose Iceberg catalog it is talking to. It needs
|
||||
`/v1/config?warehouse=`, `/v1/{prefix}/namespaces`, `/v1/{prefix}/namespaces/{ns}/tables`
|
||||
and unit-separator (`\x1F`) multi-level namespaces. We serve all of those, and
|
||||
`parseNamespace` in `weed/s3api/iceberg/utils.go:22` already splits on `\x1F`. So a stock
|
||||
Lance client pointed at our Iceberg catalog on :8181 with the Iceberg impl should already
|
||||
create, list, describe and deregister Lance tables today, with no SeaweedFS change at all.
|
||||
|
||||
That is worth testing before writing a line of the design above, for two reasons. It is a
|
||||
free baseline — and possibly a free announcement. And it is a data-loss hazard.
|
||||
|
||||
A Lance table registered this way is an Iceberg table whose metadata references no data
|
||||
files, sitting on top of a Lance dataset that uses `data/` for its fragments — the same
|
||||
subdirectory name Iceberg uses. The maintenance worker's orphan cleaner walks exactly
|
||||
`<table>/metadata` and `<table>/data`, and deletes every file not referenced by a snapshot
|
||||
and older than `orphan_older_than_hours`
|
||||
(`weed/worker/tasks/iceberg/operations.go:331`, default 72). Against an adapter-registered
|
||||
Lance table, every fragment is unreferenced by construction. Run maintenance and the
|
||||
dataset is deleted.
|
||||
|
||||
Maintenance is disabled by default (`handler.go:334`), so this is a latent hazard rather
|
||||
than a live one: it needs an operator to enable Iceberg maintenance on a bucket that also
|
||||
holds adapter-registered Lance tables. But it costs nothing to close — detection should
|
||||
skip any table carrying a non-Iceberg format marker (`table_type` property, or
|
||||
`Format != "ICEBERG"` once the format field is honest), and that guard is worth landing on
|
||||
its own regardless of whether the rest of this design ever gets built. It is the same
|
||||
"catalog-only, no maintenance" marker the generic-format question needs.
|
||||
|
||||
## Where we differ from Gravitino
|
||||
|
||||
Gravitino is a metadata service in front of somebody else's object store:
|
||||
|
||||
```
|
||||
Spark / Ray Spark / Ray / pandas / duckdb
|
||||
| |
|
||||
Lance REST Lance REST (direct S3)
|
||||
| | |
|
||||
Gravitino SeaweedFS S3 gateway ----+
|
||||
| |
|
||||
S3 keys handed out SeaweedFS filer + volumes
|
||||
|
|
||||
somebody else's S3
|
||||
```
|
||||
|
||||
It resolves a name to a location plus `lance.storage.*` credentials, and steps out of the
|
||||
way. Everything a Lance table actually is — `_versions/`, `data/`, `_indices/` — is opaque
|
||||
to it.
|
||||
|
||||
We are the store. Three things follow that Gravitino cannot do:
|
||||
|
||||
1. The catalog and a plain directory listing can be made to agree, so a client with no
|
||||
catalog at all still sees the right tables.
|
||||
2. `_versions/` is a filer directory listing, not an object-store `LIST`. Version history
|
||||
is cheap and can back the admin UI.
|
||||
3. We can offer a genuinely atomic commit reservation. Lance's commit protocol needs
|
||||
put-if-not-exists; our S3 layer does not currently provide one (see
|
||||
[Commit safety](#commit-safety)). The filer does.
|
||||
|
||||
## Placement
|
||||
|
||||
The Iceberg catalog is a thin HTTP shell over `s3tables.Manager`; the storage work lives in
|
||||
`weed/s3api/s3tables`. Table buckets live under `TablesPath = s3_constants.DefaultBucketsPath`,
|
||||
i.e. the same filer tree the S3 gateway serves, so `s3://bucket/ns/table/` is simultaneously
|
||||
a catalog entry and an S3 prefix. Catalog entries are filer directories carrying `s3tables.*`
|
||||
extended attributes. `Table.Format` already exists and is hard-checked against `"ICEBERG"`
|
||||
in `weed/s3api/s3tables/handler_table.go:48`.
|
||||
|
||||
So:
|
||||
|
||||
```
|
||||
weed/s3api/lance/ new: HTTP surface, id codec, error model
|
||||
weed/s3api/s3tables/ extended: Format "LANCE", lance state xattr, version entries
|
||||
weed/command/s3.go new: -port.lance (default 9101), startLanceServer
|
||||
```
|
||||
|
||||
`Format: "LANCE"` on the table entry is the whole storage-model change for phase 1.
|
||||
Everything else — namespaces, ARNs, policies, tags, ownership — is shared verbatim.
|
||||
|
||||
```
|
||||
s3tables.Manager (filer)
|
||||
|
|
||||
+------------------------+------------------------+
|
||||
| |
|
||||
weed/s3api/iceberg weed/s3api/lance
|
||||
Iceberg REST :8181 Lance REST :9101
|
||||
| |
|
||||
Iceberg tables Lance datasets
|
||||
\ /
|
||||
+-------------------- s3 :8333 -----------------+
|
||||
|
|
||||
SeaweedFS volumes
|
||||
```
|
||||
|
||||
## Identifier mapping
|
||||
|
||||
Lance identifiers are `["ns", ..., "table"]`, encoded in the URL as a single string joined
|
||||
by a delimiter that defaults to `$`. The delimiter alone means the root namespace, so
|
||||
`/v1/namespace/$/list` lists the root's children.
|
||||
|
||||
Iceberg had to invent a warehouse selector because its identifier is flat and every table
|
||||
bucket is a separate catalog. Lance does not need that — its identifier is already
|
||||
hierarchical, and Gravitino uses exactly three levels (`["lance_catalog", "sales", "orders"]`).
|
||||
That maps onto us without inventing anything:
|
||||
|
||||
```
|
||||
$ root -> list of table buckets
|
||||
$analytics level 1 -> a table bucket
|
||||
$analytics$sales level 2 -> a namespace in that bucket
|
||||
$analytics$sales$orders table
|
||||
```
|
||||
|
||||
`spark.sql.catalog.lance.parent = analytics` then makes `sales.orders` resolve, which is the
|
||||
same shape Gravitino's Spark example uses.
|
||||
|
||||
Levels 2..N join into one `s3tables` namespace with `.`, matching what the Iceberg catalog
|
||||
already does with `flattenNamespacePath`. The flattened form is only the directory name —
|
||||
`namespaceMetadata.Namespace []string` in the xattr keeps the authoritative parts, so the
|
||||
mapping stays invertible even though `.` is a legal character inside a namespace part.
|
||||
Reject `$` in any name part with `InvalidInput`; our charsets already exclude it, so no
|
||||
escaping scheme is needed.
|
||||
|
||||
Root-level `ListNamespaces` returning table buckets means an unauthenticated or
|
||||
broadly-scoped caller can enumerate buckets. Filter it through the same
|
||||
`s3tables/permissions.go` check `ListTableBuckets` uses, not a separate path.
|
||||
|
||||
`CreateNamespace` on a one-part identifier creates a table bucket, and it does so only if
|
||||
the caller is permitted to — the namespace never creates a bucket as a side effect of
|
||||
creating something inside it. A table bucket is a tenant resource with its own policy, ARN
|
||||
and lifecycle, and conjuring one because a client said `CREATE SCHEMA` is a privilege
|
||||
escalation dressed as a convenience. Lakekeeper draws the same line explicitly: its client
|
||||
creates tables, not warehouses.
|
||||
|
||||
## Storage layout
|
||||
|
||||
Lay tables out as:
|
||||
|
||||
```
|
||||
s3://<table-bucket>/<flattened-namespace>/<table>/
|
||||
data/
|
||||
_versions/
|
||||
_indices/
|
||||
```
|
||||
|
||||
**Built without the `.lance` suffix this design originally proposed.** The suffix would have
|
||||
made every namespace prefix a valid Lance Directory Catalog V1 root, since V1 recognises a
|
||||
table by exactly that naming. It does not survive contact with the storage layer: the
|
||||
catalog entry *is* the dataset directory, `validateTableName` excludes `.` from the charset,
|
||||
and a suffixed entry name would leak into ARNs, policy documents and the S3 Tables API,
|
||||
where the same table would answer to two different names. Making `GetTablePath` format-aware
|
||||
instead spreads an "unless it is Lance" branch through code that has no business knowing —
|
||||
the exact cross-cutting cost this design rejects family 3 for.
|
||||
|
||||
So one name, one directory. What survives is direct access by URI, which is the larger half
|
||||
of the story and needs no naming convention at all:
|
||||
|
||||
```python
|
||||
# with the catalog
|
||||
spark.sql("SELECT * FROM lance.sales.orders")
|
||||
|
||||
# without it, same bytes
|
||||
lance.dataset("s3://analytics/sales/orders")
|
||||
```
|
||||
|
||||
DuckDB, pandas and DataFusion still reach the data with no catalog running, which is the gap
|
||||
Gravitino's documentation admits to. What they no longer get for free is *enumeration* — a
|
||||
directory-catalog client pointed at the namespace prefix will not list these as tables. If
|
||||
that turns out to matter, the cheapest fix is a repair-style tool that materialises `.lance`
|
||||
aliases, not a rename of the catalog entry.
|
||||
|
||||
Note also that the directory catalog's own V2 mode puts child-namespace tables in
|
||||
`<hash>_<ns$table>` directories at the root and creates no physical subdirectories for
|
||||
namespaces, so full directory-catalog fidelity was never on offer anyway. We are a
|
||||
server-backed catalog; the human-readable prefix layout is worth more than partial V1
|
||||
lookalike behaviour.
|
||||
|
||||
## The table bucket was not a neutral container
|
||||
|
||||
This design assumed a table bucket is a place to put a table's files. It is
|
||||
not: `validateTableBucketObjectPath` runs on every S3 write into one and
|
||||
validated the path against Iceberg's layout, so a Lance client got 403 on
|
||||
`data/*.lance`, on `_versions/`, and on `_transactions/` — a directory Lance
|
||||
writes that neither the spec documentation nor this design anticipated. Nothing
|
||||
about the catalog worked end to end until that changed.
|
||||
|
||||
The layout guard now admits the union of what the supported formats write, and
|
||||
treats any underscore-prefixed top-level directory as belonging to the format,
|
||||
checking only that the path stays inside the table. Enumerating Lance's
|
||||
internal directories by name is exactly the mistake that missed
|
||||
`_transactions`. Iceberg writes none of them, so it loses nothing.
|
||||
|
||||
Found by pointing the real Python client at a running gateway, not by reading
|
||||
the spec. Worth remembering for the next format: the premise to check first is
|
||||
whether the storage layer will accept its files at all.
|
||||
|
||||
## Table lifecycle
|
||||
|
||||
Lance has three table states, and the spec pins them to marker files:
|
||||
|
||||
| State | Marker | Created by | Visible in ListTables |
|
||||
| --- | --- | --- | --- |
|
||||
| declared | `.lance-reserved` | `DeclareTable` | yes, when `include_declared=true` |
|
||||
| created | `_versions/` present | client writes, or `CreateTable` | yes |
|
||||
| deregistered | `.lance-deregistered` | `DeregisterTable` | no; data preserved |
|
||||
|
||||
Record the state in an xattr (`s3tables.lanceState`) on the catalog entry *and* write the
|
||||
marker file into the table directory. The xattr is what the catalog reads; the marker is
|
||||
what keeps a directory-catalog client honest. Dual-write is the price of the interop claim
|
||||
above, and it is one extra filer write on three rarely-called operations.
|
||||
|
||||
`DeclareTable` is the operation `lance-spark` actually calls on `CREATE TABLE` (it replaced
|
||||
the legacy `create-empty`), so it is not optional in practice even though the spec marks
|
||||
only a subset as required.
|
||||
|
||||
`DeregisterTable` preserving data is the same shape as our Iceberg rename, where the catalog
|
||||
entry moves and the data stays put — reuse `TableDataDirFromMetadataLocation`'s idea rather
|
||||
than re-deriving the data path from the catalog name.
|
||||
|
||||
## Commit safety
|
||||
|
||||
This is the part I got wrong, and the correction removed a feature rather than adding one.
|
||||
|
||||
Lance commits a version by writing `_versions/{v}.manifest` with put-if-not-exists: exactly
|
||||
one writer is supposed to win, and the loser rebases. In lance 10 that path is not optional
|
||||
and needs nothing bolted on — `commit_handler_from_url` hands every `s3://` dataset a
|
||||
`ConditionalPutCommitHandler`, which calls `put_opts` with `PutMode::Create`, which
|
||||
object_store's S3 backend sends as `If-None-Match: *`.
|
||||
|
||||
I originally read our gateway as evaluating that header check-then-act, and designed around
|
||||
it. That was already out of date. `buildWriteCondition`
|
||||
(`weed/s3api/s3api_object_routed_write.go`) reduces `If-None-Match: *` to a filer
|
||||
`WriteCondition{IF_NOT_EXISTS}`, and `putToFiler` routes the create to the object's owner
|
||||
filer, which evaluates the precondition under its per-path lock; when routing is not
|
||||
available it falls back to the object write lock, which evaluates it under the lock too.
|
||||
Either way it is atomic. Sixteen concurrent writers of the same fresh key get one 200 and
|
||||
fifteen 412s, repeatedly.
|
||||
|
||||
So the store already has the primitive Lance needs, cluster-wide, for every conditional-PUT
|
||||
client and not just this one.
|
||||
|
||||
### What that removed
|
||||
|
||||
An earlier draft of this design offered the catalog as an **external manifest store**:
|
||||
`managed_versioning: true` plus `CreateTableVersion` and friends, with the reserve step as a
|
||||
filer `CreateEntry` with `o_excl`. It was implemented, tested, and shipped behind a default-off
|
||||
flag — and it should not exist.
|
||||
|
||||
- It solves a problem this store does not have. The spec offers that path for stores that
|
||||
cannot order commits themselves.
|
||||
- It moves a table's version history out of the dataset and into the catalog, so a reader
|
||||
that does not go through this namespace no longer sees the whole picture. That is a real
|
||||
cost paid for nothing.
|
||||
- lance 10 cannot even use it past the first commit: `NamespaceManifestStore::put_if_not_exists`
|
||||
answers "put_if_not_exists is not supported for namespace-backed stores", which is exactly
|
||||
what a second `append` needs.
|
||||
|
||||
The version operations now answer `Unsupported` alongside the other operations the catalog
|
||||
does not serve, and `managed_versioning` is answered `false`. The property they were
|
||||
protecting is covered instead by a test that races eight writers at the manifest key through
|
||||
S3 and asserts one wins — testing the path Lance actually takes.
|
||||
|
||||
## Credential vending
|
||||
|
||||
Iceberg needed a header (`X-Iceberg-Access-Delegation: vended-credentials`) and a bespoke
|
||||
response shape. Lance has it in the spec: `vend_credentials: true` on the request,
|
||||
`storage_options` on the response, with `expires_at_millis` as the well-known expiry key.
|
||||
|
||||
Reuse the existing vendor interface unchanged — `iceberg.CredentialVendor` /
|
||||
`STSService.AssumeRoleForPrincipal` scoped to the table prefix (#10777) — and map its output
|
||||
to the storage options Lance passes through to `object_store`:
|
||||
|
||||
```
|
||||
aws_access_key_id, aws_secret_access_key, aws_session_token,
|
||||
aws_region, aws_endpoint, allow_http, expires_at_millis
|
||||
```
|
||||
|
||||
Those are the names `pylakekeeper` emits as `lance_storage_options`, which is the shape
|
||||
Lakekeeper's tested S3 path actually feeds to Lance. `object_store` also accepts the
|
||||
un-prefixed aliases (`endpoint`, `region`) that the directory catalog's `storage.` prefix
|
||||
strips down to and that Gravitino's `lance.storage.endpoint` resolves to, but the `aws_`
|
||||
forms are the ones with a tested integration behind them, so emit those. `aws_endpoint`
|
||||
should come from `deriveS3AdvertisedEndpoint()`, the same source the Iceberg `FileIO` config
|
||||
uses, and `allow_http` must be set when that endpoint is plain HTTP or every read fails with
|
||||
a TLS error that looks like a credential problem — Lakekeeper vends both automatically for
|
||||
exactly this reason, and calls out that there is then no per-vendor branch in client code.
|
||||
|
||||
We emit this server-side, in the `storage_options` field the Lance spec already defines,
|
||||
which is strictly better than Lakekeeper's arrangement: no client library has to translate
|
||||
anything, so vending works from any stock Lance client rather than only from theirs.
|
||||
|
||||
Guard the same way #10777 had to after review: bucket-scoped list grants need an `s3:prefix`
|
||||
condition, and a location containing `*` or `?` must be refused rather than widened into a
|
||||
resource pattern.
|
||||
|
||||
## Auth and authorization
|
||||
|
||||
Authentication reuses `S3Authenticator` and `CredentialValidator` as-is. The Lance spec maps
|
||||
identity to headers — `api_key` to `x-api-key`, `auth_token` to `Authorization: Bearer` — and
|
||||
SigV4 keeps working because it is the same authenticator the Iceberg catalog already fronts.
|
||||
|
||||
Authorization needs nothing new. A Lance table gets the same ARN shape,
|
||||
`arn:aws:s3tables:...:bucket/B/table/NS/T`, so every existing table-bucket policy covers
|
||||
Lance tables with no new policy language and no second permission model. Route it through
|
||||
`s3tables/permissions.go` and inherit the `DefaultAllow` semantics the Iceberg server already
|
||||
mirrors from the S3 port.
|
||||
|
||||
One spec quirk worth honoring: request context entries prefixed `header.` become request
|
||||
headers, and every response header comes back as a `header.`-prefixed context entry. Echoing
|
||||
`x-request-id` through it costs nothing and makes tracing work.
|
||||
|
||||
## What to take from Lakekeeper
|
||||
|
||||
Rejecting Lakekeeper's API shape does not mean rejecting what it learned building it.
|
||||
|
||||
**Deregister is soft-delete, so implement it as one.** Lakekeeper gives generic tables
|
||||
soft-deletion with undrop and a `protected` flag that makes a drop require `force=true`.
|
||||
Lance already has the concept — `DeregisterTable` preserves the data and hides the table —
|
||||
so the `.lance-deregistered` marker is a soft-delete by another name, and a re-register is
|
||||
an undrop. A protection flag on table-bucket entries is worth having regardless of Lance:
|
||||
it is a few lines against the existing xattrs and it applies to Iceberg tables too.
|
||||
|
||||
**Enforce one identifier space across entry kinds.** Lakekeeper rejects a generic table
|
||||
whose name collides with an Iceberg table or view in the same namespace. Our catalog entries
|
||||
already share one filer directory and already carry `s3tables.entryType`, so this is
|
||||
structurally true — but it has to be enforced deliberately on every path, or a Lance handler
|
||||
happily loads an Iceberg table's directory and vice versa. That is the same crossover bug
|
||||
class as the view/table rename authorization fixed in #10776; the `catalogEntryKind` pattern
|
||||
from that change is the thing to reuse rather than re-derive.
|
||||
|
||||
**A re-vend path matters more than it looks.** Lakekeeper exposes `/credentials` separately
|
||||
from load, because STS credentials expire in the middle of long jobs and re-loading the
|
||||
whole table to refresh them is wasteful. In Lance the spec's answer is another
|
||||
`DescribeTable` with `vend_credentials: true`, which is fine — but it means `DescribeTable`
|
||||
must stay cheap when `load_detailed_metadata` is false, which is another reason not to open
|
||||
the dataset on that path.
|
||||
|
||||
**Generic tables are a cheap orthogonal win.** Lakekeeper's real insight is that Delta,
|
||||
Parquet, CSV, Vortex and Paimon all get governance for free once the catalog stops caring
|
||||
what the format is. Our `Table.Format` field already exists and the only thing stopping it
|
||||
is the hard `"ICEBERG"` check in `handler_table.go:48`. Loosening that and letting the S3
|
||||
Tables API register a table with an arbitrary format and a location — no metadata, no
|
||||
commits — is a small change that makes every format cataloguable. It is independent of this
|
||||
design and probably worth doing first, since `Format: "LANCE"` is then just a value rather
|
||||
than a special case.
|
||||
|
||||
**Skip remote signing.** It is Lakekeeper's fallback for S3-compatible stores with no STS,
|
||||
and their own documentation notes that Lance will not use it — format libraries with their
|
||||
own S3 client expect static credentials and do not implement the Iceberg signer protocol. We
|
||||
have STS, so vended credentials are the path, and the signer is not worth building for a
|
||||
client that cannot consume it.
|
||||
|
||||
## Errors
|
||||
|
||||
Lance uses `{code, error, detail, instance}` with numeric codes, not Iceberg's exception-type
|
||||
strings. The mapping is mechanical:
|
||||
|
||||
| HTTP | code | when |
|
||||
| --- | --- | --- |
|
||||
| 400 | 13 InvalidInput | charset violations, malformed id, route/body mismatch |
|
||||
| 401 | 16 Unauthenticated | |
|
||||
| 403 | 15 PermissionDenied | |
|
||||
| 404 | 1 NamespaceNotFound, 4 TableNotFound, 11 TableVersionNotFound | |
|
||||
| 409 | 2/5 AlreadyExists, 3 NamespaceNotEmpty, 14 ConcurrentModification | |
|
||||
| 501 | 0 Unsupported | every phase-3 data operation |
|
||||
|
||||
Route/body mismatch is a spec requirement, not a nicety: when the identifier appears in both
|
||||
the path and the body and they disagree, the server must return 400. Cheap to get right at
|
||||
the decode step, annoying to retrofit.
|
||||
|
||||
## Route surface
|
||||
|
||||
Phase 0 is not in this table: point a stock Lance client at the existing Iceberg catalog
|
||||
with the Iceberg impl, see how far it gets, and land the maintenance guard either way. That
|
||||
tells us what the native server actually has to beat.
|
||||
|
||||
Phase 1, the whole `lance-spark` and `lance-ray` contract:
|
||||
|
||||
```
|
||||
POST /v1/namespace/{id}/create CreateNamespace mode: Create|ExistOk|Overwrite
|
||||
GET /v1/namespace/{id}/list ListNamespaces
|
||||
POST /v1/namespace/{id}/describe DescribeNamespace
|
||||
POST /v1/namespace/{id}/drop DropNamespace mode: Fail|Skip, behavior: Restrict|Cascade
|
||||
POST /v1/namespace/{id}/exists NamespaceExists
|
||||
GET /v1/namespace/{id}/table/list ListTables ?include_declared, ?page_token, ?limit
|
||||
GET /v1/table ListAllTables
|
||||
POST /v1/table/{id}/declare DeclareTable
|
||||
POST /v1/table/{id}/describe DescribeTable ?with_table_uri, ?load_detailed_metadata, ?check_declared
|
||||
POST /v1/table/{id}/exists TableExists
|
||||
POST /v1/table/{id}/register RegisterTable mode: Create|Overwrite
|
||||
POST /v1/table/{id}/deregister DeregisterTable
|
||||
POST /v1/table/{id}/drop DropTable
|
||||
POST /v1/table/{id}/rename RenameTable
|
||||
```
|
||||
|
||||
`DescribeTable` with `load_detailed_metadata=false` needs only `location`, which is the
|
||||
common case and which we can answer from xattrs alone. With `load_detailed_metadata=true`
|
||||
the spec wants `version`, `schema` and `stats`, which means reading the Lance manifest. For
|
||||
phase 1, return the fields we can derive from the filer — `version` from the highest entry in
|
||||
`_versions/`, given V2 naming is `{u64::MAX - version:020}.manifest` and V1 is
|
||||
`{version}.manifest` — and omit `schema`/`stats` rather than fabricating them. The spec
|
||||
tolerates a partial response here; it does not tolerate a wrong one.
|
||||
|
||||
Phase 2 was the five version operations plus `managed_versioning`; it was built and then
|
||||
removed, for the reasons under Commit safety.
|
||||
|
||||
Phase 3 is the data plane: `CreateTable`, `InsertIntoTable`, `MergeInsertIntoTable`,
|
||||
`UpdateTable`, `DeleteFromTable`, `QueryTable`, `CountTableRows`, and the index and tag
|
||||
operations. These exchange Arrow IPC, and more to the point they require reading and writing
|
||||
the Lance file format, for which no Go implementation exists. Return `Unsupported` (code 0)
|
||||
and say so in the docs. `arrow-go/v18` is already an indirect dependency, so Arrow framing is
|
||||
not the blocker — Lance is.
|
||||
|
||||
## Does a Lance table need maintenance?
|
||||
|
||||
Yes, and one part of it has no Iceberg equivalent. The client exposes three jobs:
|
||||
|
||||
- `optimize.compact_files()` — Lance writes a fragment per write batch, so a table fed by
|
||||
small appends accumulates small files exactly the way an Iceberg table does.
|
||||
- `optimize.optimize_indices()` — **rows written after an index was built are not covered by
|
||||
it.** A vector search against a stale index silently misses recent data. That is a
|
||||
correctness-shaped failure, not a slow query, and it is specific to what people use Lance
|
||||
for.
|
||||
- `cleanup_old_versions()` — every version is retained until something removes it. Lance can
|
||||
do this itself: `optimize.enable_auto_cleanup()` sets it on the dataset, so this one need
|
||||
not be an external job at all.
|
||||
|
||||
None of it can run in the Go worker. All three read and rewrite Lance files, which needs
|
||||
Lance format code that does not exist in Go, and there is no useful subset either: deciding
|
||||
which fragments an old version still references means parsing Lance manifests.
|
||||
|
||||
So the maintenance worker must not touch a Lance table, and it declines by reading the format
|
||||
the catalog recorded rather than by failing to parse Iceberg metadata.
|
||||
|
||||
## The worker can be Rust, and it is not a sidecar
|
||||
|
||||
The Go worker is not the only worker. `weed/pb/plugin.proto` defines `PluginControlService`,
|
||||
a language-agnostic gRPC stream that external maintenance workers connect on: the worker
|
||||
opens `WorkerStream`, sends `WorkerHello` with the job types it can `detect` and `execute`,
|
||||
answers `RequestConfigSchema` with a `JobTypeDescriptor`, replies to `RunDetectionRequest`
|
||||
with `JobProposal`s and to `ExecuteJobRequest` with `JobProgressUpdate`s and `JobCompleted`.
|
||||
`weed worker -admin=host:23646` is the Go reference implementation of exactly that contract,
|
||||
from outside the admin process.
|
||||
|
||||
Nothing in it is Go-specific, and the Rust toolchain is already in the tree.
|
||||
`seaweed-volume/build.rs` compiles protos straight out of `../weed/pb/` with `tonic_build`,
|
||||
including `filer.proto`, on tonic 0.12 and prost 0.13. A Lance worker is that same build
|
||||
with `plugin.proto` added and the `lance` crate as a dependency — the real one, no FFI and
|
||||
no Python.
|
||||
|
||||
Three job types, one per real maintenance operation:
|
||||
|
||||
| Job type | Calls | Detected from |
|
||||
| --- | --- | --- |
|
||||
| `lance_compact` | `optimize.compact_files` | fragment count and sizes |
|
||||
| `lance_optimize_indices` | `optimize.optimize_indices` | rows an index does not cover |
|
||||
| `lance_cleanup_versions` | `cleanup_old_versions` | version count and age |
|
||||
|
||||
What the existing machinery then supplies for free is the part worth noticing. Scheduling,
|
||||
retries, dedupe by `dedupe_key`, progress reporting, per-job concurrency limits and the
|
||||
admin settings page all come from the protocol: a worker that answers `RequestConfigSchema`
|
||||
with a descriptor gets its configuration form rendered in the admin UI without a line of Go
|
||||
or templ. A Rust worker is a first-class maintenance worker, not an appendage.
|
||||
|
||||
The remaining wiring is small and mostly decided already. `RunDetectionRequest` carries a
|
||||
`ClusterContext` with filer and S3 addresses plus a free-form `metadata` map, which is where
|
||||
the Lance namespace URL goes; the worker lists Lance tables from the namespace, which is the
|
||||
catalog of record and already filters by format. It gets at the data by asking
|
||||
`DescribeTable` for `storage_options` with `vend_credentials`, so the worker is just another
|
||||
client of the STS path rather than a component with its own credentials. And when it commits
|
||||
a compaction it goes through `CreateTableVersion` like any other writer, which is what
|
||||
managed versioning was for.
|
||||
|
||||
## The worker is also the only thing that can describe the table
|
||||
|
||||
Admin can render an Iceberg table because it can read Iceberg metadata. It cannot read
|
||||
Lance: it knows the dataset's location and its format string, and that is the whole of it.
|
||||
The details page showed a location and two empty panels, which is an honest answer and a
|
||||
useless one.
|
||||
|
||||
The worker already knows. Detection opens every dataset to decide whether it needs
|
||||
compacting, so at that moment it holds the schema, the row count, the fragment count and
|
||||
the version count. It just had no way to say so — every message on the stream was about
|
||||
work.
|
||||
|
||||
So `WorkerObservations` is a body on `WorkerToAdminMessage`: a repeated `ObjectObservation`
|
||||
of `object_id`, `object_kind`, `format`, and a `ConfigValue` map the worker fills with
|
||||
whatever it can cheaply say. Admin keeps the last observation per object and serves it back
|
||||
with the time it was taken and the worker that took it. Nothing schedules from it, and it is
|
||||
not authoritative — it is a cache with its staleness on the label, which is why the page
|
||||
badges it rather than presenting it as metadata it read itself.
|
||||
|
||||
The keys are the worker's to choose, which keeps the protocol out of the business of knowing
|
||||
what a Lance table is. A worker for any other format admin cannot parse describes itself the
|
||||
same way.
|
||||
|
||||
## A bucket declares its format
|
||||
|
||||
Format was recorded per table, which is enough for the storage layer and not enough for
|
||||
anything that has to answer a question about a bucket. The admin UI printed one Iceberg
|
||||
endpoint for every bucket, including the ones holding Lance datasets, where that endpoint
|
||||
serves nothing; an empty bucket had no format at all.
|
||||
|
||||
So `CreateTableBucket` takes an optional `format`, stored with the rest of the bucket
|
||||
metadata. Empty means `ICEBERG` - what AWS S3 Tables serves, and therefore what an SDK
|
||||
that has never heard of the field means. `CreateTable` refuses another format, and
|
||||
`CreateView` refuses outright outside an Iceberg bucket, a view being Iceberg metadata.
|
||||
The Lance namespace declares `LANCE` for the buckets it creates.
|
||||
|
||||
**Enforced rather than defaulted**, because the point of showing a format at all is the
|
||||
endpoint that follows from it, and that endpoint is only truthful if the bucket holds one
|
||||
format. **Buckets that already exist stay undeclared** and keep taking anything: nothing is
|
||||
migrated, and the UI shows "unset" as a fact about the bucket's age rather than a fault.
|
||||
That state is also the only way to hold both formats at once, which is what the
|
||||
Iceberg-REST adapter path produces.
|
||||
|
||||
## Sample rows are fetched, not cached
|
||||
|
||||
The same asymmetry has a second half. Admin renders an Iceberg table's rows by
|
||||
reading its Parquet files directly; for Lance it has nothing to read with, so the data
|
||||
page offered a Browse Data button that led to an empty grid.
|
||||
|
||||
`RequestObjectPreview` / `ObjectPreviewResponse` mirror the config-schema round trip
|
||||
already on the stream: admin asks, the worker scans the dataset and hands back rows it
|
||||
has already rendered as text, because it is the only side that knows the types. Admin
|
||||
picks the worker from the observation store, so the one that last described a table is
|
||||
the one asked to read it.
|
||||
|
||||
The rows are deliberately not cached, and that is the line between the two channels. An
|
||||
observation describes an object, so a copy with a timestamp on it is useful. Rows are the
|
||||
object's contents: a copy held in admin would be stale, larger, and nobody's business.
|
||||
The page fetches on load, bounded, or says why it cannot.
|
||||
|
||||
## The sidecar question
|
||||
|
||||
The data plane is a different problem, and this design previously conflated the two.
|
||||
Maintenance rides the worker protocol; `QueryTable` and `InsertIntoTable` do not, because
|
||||
they are synchronous REST operations on the namespace's own surface. Serving those means a
|
||||
Rust process that answers HTTP, either behind the Go namespace as a proxy target or in front
|
||||
of it. It would make SeaweedFS a store you can run vector search *in* rather than one you
|
||||
read vectors *out of*, which is the larger prize and the reason to keep the option open.
|
||||
|
||||
Neither should gate phase 1. Phases 1 and 2 are pure Go over the filer and are worth
|
||||
shipping on their own — they are what makes Spark and Ray work.
|
||||
|
||||
## Testing
|
||||
|
||||
Mirror the Iceberg package: `httptest` plus a fake filer client for the handler tests, in
|
||||
`weed/s3api/lance`. Then an integration suite under `test/s3tables/catalog/` next to the
|
||||
existing `pyiceberg_test.go`, driving the generated Python `lance-namespace` client against
|
||||
a live gateway. Three things that suite must cover and unit tests cannot:
|
||||
|
||||
- the storage-options key names actually work, i.e. a client that gets `storage_options` from
|
||||
`DescribeTable` can open the dataset;
|
||||
- a table created through the catalog is visible to `lance.dataset()` by URI and to a V1
|
||||
directory-catalog client rooted at the namespace prefix;
|
||||
- concurrent writers do not lose a commit, which is the phase-2 acceptance test and the
|
||||
thing that justifies the external manifest store.
|
||||
|
||||
Phase 1 is validated: `lance_namespace` 0.11.1 with `impl=rest` drives the namespace,
|
||||
`lance.write_dataset` writes to the vended location with the vended `storage_options`, and
|
||||
the rows read back. Note that this client version drops `check_declared` and
|
||||
`include_declared` on the wire, so `is_only_declared` reads null through it however the
|
||||
server behaves.
|
||||
|
||||
The commit path is validated at both levels. The mechanism: eight writers race the same
|
||||
manifest key through S3 with `If-None-Match: *`, and exactly one wins. The property that
|
||||
actually matters, which single-winner exclusivity does not by itself establish: eight
|
||||
writers append to one dataset concurrently through lance, and afterwards every batch is
|
||||
still there — the losers saw the conflict, rebased, and committed again. That second test
|
||||
is also the sequence managed versioning could not complete at all, since its store answers
|
||||
"put_if_not_exists is not supported" to the second commit.
|
||||
|
||||
One more that belongs in the Iceberg suite, not this one: a Lance dataset registered through
|
||||
the Iceberg adapter must survive a full maintenance pass. Reading the code, that test should
|
||||
fail today; it has not been run.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Root-level `ListNamespaces` enumerating table buckets is convenient and is a listing
|
||||
surface we do not have on the Iceberg side. Decide whether it is gated behind a flag.
|
||||
- Whether the `.lance` directory suffix is worth the divergence from the Iceberg layout. I
|
||||
think yes — it is what makes the catalog optional — but it means the two catalogs' tables
|
||||
do not look alike on disk, and the admin UI has to know that.
|
||||
- Names: our charsets are lowercase-only and Lance identifiers are arbitrary strings. Reject
|
||||
and document, as Iceberg does, or case-fold. Rejecting is right, but see #10734 for how
|
||||
case handling bites when only one side normalizes.
|
||||
- Whether to land generic-format registration first. Dropping the `"ICEBERG"` check and
|
||||
letting a table carry an arbitrary format plus a location is smaller than this whole
|
||||
design, gets Delta and Parquet catalogued as a side effect, and turns `Format: "LANCE"`
|
||||
into an ordinary value. The argument against is that it invites tables the maintenance
|
||||
worker cannot service, so it needs a "catalog-only, no maintenance" marker to be honest.
|
||||
+15
-11
@@ -2,21 +2,25 @@ FROM ubuntu:22.04
|
||||
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
|
||||
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
|
||||
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
|
||||
# apt-install prefers Azure's mirror, which archive.ubuntu.com is slow enough from
|
||||
# GitHub-hosted runners to justify, and falls through to archive.ubuntu.com when
|
||||
# Azure is unreachable - which it periodically is, and Acquire::Retries against a
|
||||
# single mirror just retries a dead host. Images built FROM this one install
|
||||
# through it for the same reason.
|
||||
COPY apt-install /usr/local/bin/apt-install
|
||||
RUN chmod +x /usr/local/bin/apt-install && \
|
||||
apt-install curl fio fuse ca-certificates && \
|
||||
rm -rf /tmp/* /var/tmp/*
|
||||
|
||||
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
|
||||
apt-get -o Acquire::http::Timeout=15 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
fio \
|
||||
fuse \
|
||||
ca-certificates \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& rm -rf /tmp/* \
|
||||
&& rm -rf /var/tmp/*
|
||||
RUN mkdir -p /etc/seaweedfs /data/filerldb2
|
||||
|
||||
COPY ./weed /usr/bin/
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26 AS builder
|
||||
FROM golang:1.25 AS builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y build-essential wget ca-certificates && \
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary,
|
||||
# so arm64/arm/386 targets skip QEMU emulation of the whole compile.
|
||||
FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
@@ -14,12 +12,9 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
|
||||
git checkout $BRANCH) || \
|
||||
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
|
||||
echo "Available branches:" && git branch -a && exit 1))
|
||||
ARG TARGETOS TARGETARCH TARGETVARIANT
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& export GOOS=$TARGETOS GOARCH=$TARGETARCH \
|
||||
&& case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \
|
||||
&& CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed .
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
# Rust volume server: use pre-built binary from CI when available (placed in
|
||||
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
|
||||
@@ -29,7 +24,6 @@ FROM alpine:3.23 as rust_builder
|
||||
ARG TARGETARCH
|
||||
ARG TAGS
|
||||
COPY weed-volume-prebuilt/ /prebuilt/
|
||||
COPY weed-worker-prebuilt/ /prebuilt-worker/
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
|
||||
WORKDIR /build/seaweed-volume
|
||||
@@ -48,30 +42,16 @@ RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
|
||||
echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
|
||||
touch /weed-volume; \
|
||||
fi
|
||||
# The Rust maintenance worker is taken pre-built or not at all: the lance jobs
|
||||
# it carries pull in arrow and datafusion, a far larger dependency tree than
|
||||
# the image build can carry, so an architecture CI did not build for gets the
|
||||
# same empty placeholder the entrypoint refuses to exec.
|
||||
RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \
|
||||
echo "Using pre-built Rust worker for ${TARGETARCH}" && \
|
||||
cp "/prebuilt-worker/weed-worker-${TARGETARCH}" /weed-worker; \
|
||||
else \
|
||||
echo "No pre-built Rust worker for ${TARGETARCH}" && \
|
||||
touch /weed-worker; \
|
||||
fi
|
||||
|
||||
# Pre-built binaries arrive via GitHub Actions artifacts, which drop the
|
||||
# executable bit, so the copied file is 0644 and exec fails with "Permission
|
||||
# denied". Restore it (no-op for the empty placeholders, which stay size 0).
|
||||
RUN chmod 0755 /weed-volume /weed-worker
|
||||
# denied". Restore it (no-op for the empty placeholder, which stays size 0).
|
||||
RUN chmod 0755 /weed-volume
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
# Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms)
|
||||
COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
|
||||
# Same for the Rust maintenance worker, which serves Lance table buckets
|
||||
COPY --from=rust_builder /weed-worker /usr/bin/weed-worker
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26 AS builder
|
||||
FROM golang:1.25 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26 AS builder
|
||||
FROM golang:1.25 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -15,9 +15,3 @@ COPY tarantool /opt/tarantool/app
|
||||
# build app
|
||||
RUN tt build app
|
||||
|
||||
# the base image's default healthcheck assumes a single-instance layout and
|
||||
# checks a control socket path that doesn't exist for this multi-instance
|
||||
# app, so it never reports healthy; check instance status instead.
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=6 \
|
||||
CMD tt status app | awk 'NR>2{if ($2!="RUNNING") exit 1} END{if (NR<=2) exit 1}'
|
||||
|
||||
|
||||
@@ -127,9 +127,6 @@ test_tarantool: tags = tarantool
|
||||
test_tarantool: build_tarantool_dev_env build
|
||||
docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
|
||||
|
||||
test_keycloak_s3: build
|
||||
docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
|
||||
|
||||
clean:
|
||||
rm ./weed
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Install packages, falling through to the next Ubuntu mirror when one is
|
||||
# unreachable. See docker/Dockerfile.e2e.
|
||||
set -e
|
||||
|
||||
# Every rewrite starts from the pristine list, so a mirror that just failed does
|
||||
# not become the pattern the next rewrite has to match.
|
||||
[ -f /etc/apt/sources.list.orig ] || cp /etc/apt/sources.list /etc/apt/sources.list.orig
|
||||
|
||||
for mirror in azure.archive.ubuntu.com archive.ubuntu.com; do
|
||||
# Any archive host, so this works whether the pristine list came from the
|
||||
# base image (archive.ubuntu.com) or a CI runner (azure.archive.ubuntu.com).
|
||||
sed "s|http://[a-z0-9.]*archive\.ubuntu\.com/ubuntu|http://$mirror/ubuntu|g; s|http://security\.ubuntu\.com/ubuntu|http://$mirror/ubuntu|g" \
|
||||
/etc/apt/sources.list.orig > /etc/apt/sources.list
|
||||
if apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=15 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=15 install -y \
|
||||
--no-install-recommends --no-install-suggests "$@"; then
|
||||
apt-get clean
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
exit 0
|
||||
fi
|
||||
echo "apt: $mirror unreachable, trying the next mirror" >&2
|
||||
done
|
||||
|
||||
exit 1
|
||||
@@ -1,46 +0,0 @@
|
||||
services:
|
||||
keycloak:
|
||||
image: quay.io/keycloak/keycloak:26.0.7
|
||||
command: ["start-dev", "--import-realm"]
|
||||
environment:
|
||||
KC_BOOTSTRAP_ADMIN_USERNAME: admin
|
||||
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
|
||||
KC_HTTP_ENABLED: "true"
|
||||
KC_HOSTNAME: "http://keycloak:8080"
|
||||
KC_HOSTNAME_STRICT: "false"
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command:
|
||||
- -v=9
|
||||
- server
|
||||
- -ip=s3
|
||||
- -filer
|
||||
- -master.volumeSizeLimitMB=16
|
||||
- -volume
|
||||
- -volume.max=0
|
||||
- -volume.preStopSeconds=1
|
||||
- -s3
|
||||
- -s3.port=8333
|
||||
- -s3.config=/etc/seaweedfs/s3_config.json
|
||||
- -s3.iam.config=/etc/seaweedfs/iam_config.json
|
||||
environment:
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
|
||||
volumes:
|
||||
- ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
|
||||
- ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
|
||||
ports:
|
||||
- "8333:8333"
|
||||
depends_on:
|
||||
keycloak:
|
||||
condition: service_healthy
|
||||
@@ -90,16 +90,6 @@ case "$1" in
|
||||
exec /usr/bin/weed-volume $ARGS $@
|
||||
;;
|
||||
|
||||
'worker-rust')
|
||||
shift
|
||||
if [ ! -s /usr/bin/weed-worker ]; then
|
||||
echo "Error: Rust maintenance worker is not available on this platform ($(uname -m))." >&2
|
||||
echo "Use 'worker' for the Go maintenance worker instead." >&2
|
||||
exit 1
|
||||
fi
|
||||
exec /usr/bin/weed-worker "$@"
|
||||
;;
|
||||
|
||||
'server')
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
|
||||
if isArgPassed "-volume.max" "$@"; then
|
||||
|
||||
@@ -11,8 +11,4 @@ scrape_configs:
|
||||
- 'master:9324'
|
||||
- 'volume:9325'
|
||||
- 'filer:9326'
|
||||
- 's3:9327'
|
||||
# A plugin worker publishes its own metrics when started with
|
||||
# -metricsPort (Go) or --metrics-port (Rust). 9328 continues the
|
||||
# series, since 9327 is already the S3 gateway's.
|
||||
# - 'worker:9328'
|
||||
- 's3:9327'
|
||||
@@ -46,7 +46,7 @@ local filer_metadata = {
|
||||
delete_by_directory_idx = function(directory)
|
||||
local space = box.space.filer_metadata
|
||||
local index = space.index.directory_idx
|
||||
-- for each found directory
|
||||
-- for each finded directories
|
||||
for _, tuple in index:pairs({ directory }, { iterator = 'EQ' }) do
|
||||
space:delete({ tuple[1], tuple[3] })
|
||||
end
|
||||
@@ -64,7 +64,7 @@ local filer_metadata = {
|
||||
end
|
||||
-- init results
|
||||
local results = {}
|
||||
-- for each found directory
|
||||
-- for each finded directories
|
||||
for _, tuple in directory_idx:pairs({ dirPath }, { iterator = 'EQ' }) do
|
||||
-- filter by name
|
||||
if filter_filename_func(tuple[3]) then
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.26
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.51.0
|
||||
cloud.google.com/go/storage v1.64.0
|
||||
cloud.google.com/go/pubsub v1.50.2
|
||||
cloud.google.com/go/storage v1.62.1
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/eapache/go-resiliency v1.6.0 // indirect
|
||||
@@ -24,7 +25,7 @@ require (
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.17.0
|
||||
github.com/go-redsync/redsync/v4 v4.16.0
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/go-zookeeper/zk v1.0.4 // indirect
|
||||
github.com/golang/protobuf v1.5.4
|
||||
@@ -32,24 +33,24 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.19.1
|
||||
github.com/klauspost/reedsolomon v1.14.1
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/klauspost/reedsolomon v1.14.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.8
|
||||
github.com/mailru/easyjson v0.9.2 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/olivere/elastic/v7 v7.0.32
|
||||
@@ -58,14 +59,14 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.21.1
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/procfs v0.20.1
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.2.0
|
||||
github.com/seaweedfs/raft v1.1.8
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
@@ -84,56 +85,54 @@ require (
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.7.1
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.46.0
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.46.0
|
||||
golang.org/x/crypto v0.55.0
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
|
||||
golang.org/x/image v0.44.0
|
||||
golang.org/x/net v0.58.0
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.52.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/image v0.39.0
|
||||
golang.org/x/net v0.54.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
golang.org/x/tools v0.48.0 // indirect
|
||||
golang.org/x/sys v0.45.0
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/tools v0.44.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.293.0
|
||||
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect
|
||||
google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6
|
||||
google.golang.org/api v0.278.0
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.56.0
|
||||
modernc.org/sqlite v1.49.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.33.0
|
||||
cloud.google.com/go/kms v1.31.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.2
|
||||
github.com/a-h/templ v0.3.1020
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.2
|
||||
github.com/apache/iceberg-go v0.6.0
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.1001
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
|
||||
github.com/apache/iceberg-go v0.5.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.33
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.34
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/cognusion/imaging v1.0.4
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/cognusion/imaging v1.0.3
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.9.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
@@ -141,25 +140,26 @@ require (
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.32.0
|
||||
github.com/pkg/sftp v1.13.11
|
||||
github.com/rabbitmq/amqp091-go v1.13.0
|
||||
github.com/rclone/rclone v1.75.0
|
||||
github.com/parquet-go/parquet-go v0.30.1
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.11.0
|
||||
github.com/rclone/rclone v1.74.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.21.0
|
||||
github.com/schollz/progressbar/v3 v3.19.1
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.4
|
||||
github.com/shirou/gopsutil/v4 v4.26.6
|
||||
github.com/tarantool/go-option v1.1.0
|
||||
github.com/tarantool/go-tarantool/v3 v3.0.0
|
||||
github.com/testcontainers/testcontainers-go v0.43.0
|
||||
github.com/redis/go-redis/v9 v9.19.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.3
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.4
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.147.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.11
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
)
|
||||
@@ -170,26 +170,25 @@ require (
|
||||
atomicgo.dev/cursor v0.2.0 // indirect
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v1.2.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.6.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.9.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v3 v3.4.1 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.1 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.7.0 // indirect
|
||||
github.com/apache/thrift v0.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.4 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.23.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
github.com/buger/jsonparser v1.2.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
|
||||
@@ -201,52 +200,53 @@ require (
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.4 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/dave/dst v0.27.2 // indirect
|
||||
github.com/diskfs/go-diskfs v1.9.4 // indirect
|
||||
github.com/diskfs/go-diskfs v1.7.0 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/docker v28.5.2+incompatible // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.9.0 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/gookit/color v1.6.0 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/gopherjs/gopherjs v1.17.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/hamba/avro/v2 v2.31.0 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jaegertracing/jaeger v1.47.0 // indirect
|
||||
github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect
|
||||
github.com/jtolds/gls v4.20.0+incompatible // indirect
|
||||
github.com/klauspost/asmfmt v1.3.2 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lib/pq v1.12.0 // indirect
|
||||
github.com/lib/pq v1.11.1 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/lpar/calendar v0.2.0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/api v1.54.2 // indirect
|
||||
github.com/moby/moby/client v0.4.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/go-archive v0.1.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.0 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.1 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
@@ -255,20 +255,19 @@ require (
|
||||
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.83 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.4 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.3 // indirect
|
||||
github.com/rogpeppe/go-internal v1.15.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
github.com/sony/gobreaker/v2 v2.4.0 // indirect
|
||||
github.com/stretchr/objx v0.5.3 // indirect
|
||||
github.com/substrait-io/substrait v0.87.0 // indirect
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.1 // indirect
|
||||
github.com/substrait-io/substrait-protobuf/go v0.85.0 // indirect
|
||||
github.com/twmb/avro v1.7.2 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/substrait-io/substrait v0.81.0 // indirect
|
||||
github.com/substrait-io/substrait-go/v7 v7.4.0 // indirect
|
||||
github.com/substrait-io/substrait-protobuf/go v0.81.0 // indirect
|
||||
github.com/twpayne/go-geom v1.6.1 // indirect
|
||||
github.com/twpayne/go-kml/v3 v3.2.1 // indirect
|
||||
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
|
||||
@@ -278,117 +277,122 @@ require (
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.38.0 // indirect
|
||||
golang.org/x/mod v0.35.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.2 // indirect
|
||||
cloud.google.com/go/auth v0.23.0 // indirect
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.11.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.29.0 // indirect
|
||||
cloud.google.com/go/iam v1.7.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.12.0 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.2 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.36 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.4
|
||||
github.com/aws/smithy-go v1.27.7
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.4 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.16.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/elastic/gosigar v0.14.3 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.3.1 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.8 // indirect
|
||||
github.com/go-openapi/strfmt v0.27.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.3 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/goccy/go-json v0.10.6 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.20 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
@@ -406,23 +410,24 @@ require (
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/k0kubun/pp v3.0.1+incompatible
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.24 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.22 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.49.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
||||
github.com/nats-io/nats.go v1.48.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.12 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
@@ -430,20 +435,20 @@ require (
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.121.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.12.1 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.5 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.28
|
||||
github.com/pierrec/lz4/v4 v4.1.26
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/pkg/xattr v0.4.12 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
@@ -451,79 +456,70 @@ require (
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/samber/lo v1.53.0 // indirect
|
||||
github.com/samber/lo v1.52.0 // indirect
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.3 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
github.com/unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260428144813-1c07baab7f7b // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.5.0 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.7.1 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.45.0
|
||||
golang.org/x/term v0.43.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.74.4 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260629224719-ba1bff0a7846 // indirect
|
||||
storj.io/drpc v1.0.0 // indirect
|
||||
storj.io/eventkit v0.0.0-20260716074419-6861a92e2aa5 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
storj.io/picobuf v0.0.4 // indirect
|
||||
storj.io/uplink v1.14.3 // indirect
|
||||
storj.io/uplink v1.14.0 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
// apache/thrift v0.23.0 fixes CVE-2026-41602 but compares int against the
|
||||
// untyped math.MaxUint32 in lib/go/thrift/framed_transport.go, which overflows
|
||||
// int on 32-bit GOARCHes (e.g. openbsd/arm, linux/arm) and fails to compile.
|
||||
// Upstream fixed the range check post-release; pin to that commit until the
|
||||
// next tagged release carries both the CVE fix and the 32-bit fix.
|
||||
replace github.com/apache/thrift => github.com/apache/thrift v0.23.1-0.20260429145742-d2acd3c49e58
|
||||
|
||||
// tyler-smith/go-bip39 was deleted from GitHub, so `go mod download` fails for
|
||||
// anyone resolving it directly (GOPROXY=direct). It only reaches us transitively
|
||||
// through rclone's internxt backend, which calls IsMnemonicValid and NewSeed.
|
||||
// cosmos/go-bip39 is a maintained, API-compatible fork.
|
||||
replace github.com/tyler-smith/go-bip39 => github.com/cosmos/go-bip39 v1.0.0
|
||||
// apache/thrift v0.23.0 uses math.MaxUint32 as an untyped int constant in
|
||||
// lib/go/thrift/framed_transport.go, which overflows int on 32-bit GOARCHes
|
||||
// (e.g. openbsd/arm, linux/arm). Pin to v0.22.0 until upstream fixes it.
|
||||
replace github.com/apache/thrift => github.com/apache/thrift v0.22.0
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
# Usage:
|
||||
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
|
||||
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
|
||||
# curl -fsSL ... | bash -s -- --version v3.93 --dir /usr/local/bin
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, all (default: weed)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.44"
|
||||
appVersion: "4.29"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.44.0
|
||||
version: 4.29.0
|
||||
|
||||
@@ -22,8 +22,8 @@ helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* leveldb2 is the default filer backend; a mysql-compatible database (memsql, ...) enables HA (multiple filer instances) and the backup/HA it can provide.
|
||||
* with `filer.extraEnvironmentVars.WEED_MYSQL_ENABLED` set to `"true"`, mysql user/password are created in a k8s secret (default: `<release>-seaweedfs-db-secret`) and injected to the filer with ENV. On any other store neither the secret nor the `WEED_MYSQL_*` env, plain or secret-backed, is rendered.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide.
|
||||
* mysql user/password are created in a k8s secret (default: `<release>-seaweedfs-db-secret`) and injected to the filer with ENV.
|
||||
* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
|
||||
|
||||
## Prerequisites
|
||||
@@ -107,120 +107,6 @@ https://github.com/rancher/local-path-provisioner
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
### Master data: hostPath vs a claim
|
||||
|
||||
The master's `-mdir` holds its Raft log and snapshots, and with them the
|
||||
cluster's identity (its topology UUID). `master.data.type` defaults to
|
||||
`hostPath`, which does not follow a pod to another node: a master that is
|
||||
rescheduled comes back with an empty data directory and a brand new cluster
|
||||
UUID. With the chart's default of a single master replica there is no peer to
|
||||
recover the identity from either.
|
||||
|
||||
Putting the master's data on a claim avoids that:
|
||||
|
||||
```yaml
|
||||
master:
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "1Gi"
|
||||
storageClass: "" # empty uses the cluster's default StorageClass
|
||||
```
|
||||
|
||||
Raft state is small, so a modest claim is enough — sizing matters far more for
|
||||
volume and filer.
|
||||
|
||||
The default is left at `hostPath` for backward compatibility:
|
||||
`volumeClaimTemplates` is immutable on a StatefulSet, so flipping the type on a
|
||||
release that already exists fails, whether the chart changes the default or you
|
||||
change it yourself:
|
||||
|
||||
```text
|
||||
StatefulSet.apps "<release>-seaweedfs-master" is invalid: spec: Forbidden:
|
||||
updates to statefulset spec for fields other than 'replicas', ... are forbidden
|
||||
```
|
||||
|
||||
New installs can set the claim from the start. To move an **existing** release
|
||||
onto a claim without losing the cluster UUID, use the migration below. The
|
||||
claim has to be seeded while the master is stopped: a running master rewrites
|
||||
its Raft state, so copying into a live pod is silently undone by the next
|
||||
restart.
|
||||
|
||||
The steps below are for the chart's default of a single master
|
||||
(`master.replicas: 1`). With several master replicas, repeat steps 1, 3 and 4
|
||||
for every ordinal, or migrate one at a time and let the remaining quorum
|
||||
re-replicate.
|
||||
|
||||
Take the names from the cluster rather than assembling them — the release
|
||||
name, `nameOverride` and `fullnameOverride` all feed the chart's fullname
|
||||
helper, so `<release>-seaweedfs` is not always right:
|
||||
|
||||
```bash
|
||||
NS=<namespace>; REL=<release>
|
||||
# scope by instance as well as component: several releases can share a namespace
|
||||
STS=$(kubectl -n $NS get sts \
|
||||
-l app.kubernetes.io/instance=$REL,app.kubernetes.io/component=master \
|
||||
-o jsonpath='{.items[0].metadata.name}')
|
||||
POD=$STS-0
|
||||
# a StatefulSet names its claims <template>-<statefulset>-<ordinal>, and this
|
||||
# chart's template is data-<namespace>
|
||||
PVC=data-$NS-$STS-0
|
||||
|
||||
# 1. back up the master data directory
|
||||
kubectl -n $NS cp $POD:/data ./master-backup
|
||||
|
||||
# 2. stop the master, leaving the rest of the release running
|
||||
kubectl -n $NS delete sts $STS --cascade=orphan
|
||||
kubectl -n $NS delete pod $POD
|
||||
|
||||
# 3. create the claim the new StatefulSet will adopt, and seed it through a
|
||||
# pod that actually mounts it
|
||||
kubectl -n $NS apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: $PVC
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: seed
|
||||
spec:
|
||||
containers:
|
||||
- name: seed
|
||||
image: alpine:3.20
|
||||
command: ["sleep", "600"]
|
||||
volumeMounts:
|
||||
- name: d
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: d
|
||||
persistentVolumeClaim:
|
||||
claimName: $PVC
|
||||
EOF
|
||||
kubectl -n $NS wait --for=condition=Ready pod/seed --timeout=120s
|
||||
kubectl -n $NS cp ./master-backup/m9333 seed:/data/
|
||||
kubectl -n $NS exec seed -- ls /data/m9333 # conf, log, snapshot, state
|
||||
kubectl -n $NS delete pod seed
|
||||
|
||||
# 4. upgrade; the StatefulSet adopts the claim you created
|
||||
helm upgrade $REL seaweedfs/seaweedfs -n $NS -f values.yaml
|
||||
```
|
||||
|
||||
Confirm the UUID survived — it must match what the cluster reported before the
|
||||
migration:
|
||||
|
||||
```bash
|
||||
kubectl -n $NS exec $POD -- curl -s localhost:9333/license/status
|
||||
```
|
||||
|
||||
**Or accept a new cluster UUID** and, if you run the enterprise edition, have
|
||||
the license re-issued against it.
|
||||
|
||||
## current instances config (AIO):
|
||||
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
@@ -289,32 +175,6 @@ stringData:
|
||||
seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"snu8yoP6QAlY0ne4","secretKey":"PNzBcmeLNEdR0oviwm04NQAicOrDH1Km"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"SCigFee6c5lbi04A","secretKey":"kgFhbT38R8WUYVtiFQ1OiSVOrYr3NKku"}],"actions":["Read"]}]}'
|
||||
```
|
||||
|
||||
#### Source S3 credentials from an existing Secret
|
||||
|
||||
To keep the keys out of `values.yaml` while still letting the chart generate the
|
||||
identities file, point an identity at an existing Secret:
|
||||
|
||||
```yaml
|
||||
s3:
|
||||
enabled: true
|
||||
enableAuth: true
|
||||
credentials:
|
||||
admin:
|
||||
existingSecret: minio-root
|
||||
accessKeyKey: root-user
|
||||
secretKeyKey: root-password
|
||||
```
|
||||
|
||||
`accessKeyKey` and `secretKeyKey` default to the chart's own key names
|
||||
(`admin_access_key_id`, `admin_secret_access_key`, and the `read_` pair). The
|
||||
generated `seaweedfs_s3_config` references the keys as `${SEAWEEDFS_S3_ADMIN_ACCESS_KEY_ID}`
|
||||
and the gateway resolves them from the environment, which the chart wires up
|
||||
from the Secret. Nothing is read from the cluster at render time, so
|
||||
`helm template`, `--dry-run` and an Argo CD diff all render what an install
|
||||
applies. Rotating a key in the Secret takes effect on the next pod restart, as
|
||||
with any other environment variable. The COSI driver parses the config itself
|
||||
and does not resolve these references.
|
||||
|
||||
## Admin Component
|
||||
|
||||
The admin component provides a modern web-based administration interface for managing SeaweedFS clusters. It includes:
|
||||
@@ -505,47 +365,6 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum
|
||||
helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
|
||||
```
|
||||
|
||||
## Network Policies
|
||||
|
||||
In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
|
||||
|
||||
```bash
|
||||
helm install seaweedfs seaweedfs/seaweedfs --set networkPolicy.enabled=true
|
||||
```
|
||||
|
||||
That alone leaves outbound traffic untouched, which is enough when the namespace's default-deny only restricts ingress. If it lists `Egress` in its `policyTypes` too - the usual baseline - the components still cannot resolve DNS, and you need the second opt-in below as well.
|
||||
|
||||
Egress is separate because the chart knows where its own components live but not where your filer store, notification sink or remote tier does, and because in a namespace with no default-deny at all, adding egress rules would narrow the components from "may reach anything" to "may reach these peers":
|
||||
|
||||
```yaml
|
||||
networkPolicy:
|
||||
enabled: true
|
||||
egress:
|
||||
enabled: true
|
||||
kubeApiServer:
|
||||
# the endpoint behind the kubernetes service, not its ClusterIP
|
||||
cidrs: ["172.18.0.2/32"]
|
||||
extraEgress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: postgresql
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
```
|
||||
|
||||
`kubeApiServer.cidrs` is only demanded when something in the release actually needs the API server, which is the COSI sidecar and, on an upgrade that grows a volume PVC, the resize hook. No seaweedfs component itself speaks to it.
|
||||
|
||||
Anything reaching the release from outside - an ingress controller, a Prometheus in another namespace - goes into `networkPolicy.extraIngress`, or into `networkPolicy.components.<component>.extraIngress` for a single component. See the `networkPolicy` block in `values.yaml` for the full set.
|
||||
|
||||
Two things worth knowing before you turn this on:
|
||||
|
||||
- **Monitoring stops.** The metrics ports are admitted from release pods like every other port, so with `global.seaweedfs.monitoring.enabled` the ServiceMonitors keep scraping targets a Prometheus in another namespace can no longer reach. Nothing reports it; add the scraper's namespace to `extraIngress`.
|
||||
- **The resize hook's policy is a Helm hook.** Its Job runs before the release manifest is applied, so the policy has to be a `pre-install` hook too. Helm does not garbage-collect hook resources, so on an upgrade that grows a volume PVC the policy is created and then left behind on uninstall - delete `<release>-seaweedfs-volume-resize-hook` by hand if it bothers you.
|
||||
|
||||
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
|
||||
|
||||
## OpenShift Support
|
||||
|
||||
SeaweedFS can be deployed on OpenShift or any cluster enforcing the Kubernetes "restricted" Pod Security Standard. By default, OpenShift blocks containers that run as root or use `hostPath` volumes.
|
||||
@@ -565,39 +384,4 @@ helm install seaweedfs seaweedfs/seaweedfs \
|
||||
## Enterprise
|
||||
|
||||
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
|
||||
which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.
|
||||
|
||||
To run it, set the image and point the chart at a Secret holding the license
|
||||
file:
|
||||
|
||||
```bash
|
||||
kubectl create secret generic seaweedfs-license -n <namespace> \
|
||||
--from-file=seaweed-license.json=/path/to/seaweed-license.json
|
||||
```
|
||||
|
||||
```yaml
|
||||
global:
|
||||
seaweedfs:
|
||||
image:
|
||||
name: chrislusf/seaweedfs-enterprise
|
||||
license:
|
||||
existingSecret: seaweedfs-license
|
||||
# secretKey: seaweed-license.json # key within the Secret
|
||||
# mountPath: /etc/seaweedfs/license # directory it is mounted at
|
||||
```
|
||||
|
||||
Set the image globally rather than per component: a per-component
|
||||
`imageOverride` wins, and a cluster that mixes editions comes up looking
|
||||
healthy with enterprise features quietly off.
|
||||
|
||||
Only the master reads the license, so the Secret is mounted read-only there
|
||||
and on all-in-one (which runs `weed server -master`). It is mounted as a
|
||||
directory, not a `subPath`, so a renewed Secret reaches the running master —
|
||||
which re-reads the file periodically — without a restart.
|
||||
|
||||
The license is tied to the cluster UUID kept in the master's Raft state, so put
|
||||
`master.data` on a claim — a master that restarts onto an empty data directory
|
||||
generates a new UUID and the license stops matching. See
|
||||
[Master data](#master-data-hostpath-vs-a-claim). Check the binding with
|
||||
`kubectl exec <master-pod> -- curl -s localhost:9333/license/status`
|
||||
(`cluster_uuid` must equal `license_uuid`).
|
||||
which has a self-healing storage format with better data protection.
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
# Default configuration, kept so chart-testing still installs the chart as-is.
|
||||
@@ -1,5 +0,0 @@
|
||||
# SFTP install: the pod only becomes ready if the server loads a host key
|
||||
# from the generated sftp-ssh-secret, so this exercises the whole path.
|
||||
sftp:
|
||||
enabled: true
|
||||
enableAuth: true
|
||||
@@ -3666,291 +3666,6 @@
|
||||
],
|
||||
"title": "S3 Bucket Object Count",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 130
|
||||
},
|
||||
"id": 92,
|
||||
"panels": [],
|
||||
"title": "Filer Object Size Distribution",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Rate of new objects created, split by size range.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "objects/s",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "normal"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 131
|
||||
},
|
||||
"id": 93,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "8.1.2",
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "< 1KB",
|
||||
"refId": "A",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1KB - 100KB",
|
||||
"refId": "B",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100KB - 1MB",
|
||||
"refId": "C",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1MB - 100MB",
|
||||
"refId": "D",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100MB - 1GB",
|
||||
"refId": "E",
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "> 1GB",
|
||||
"refId": "F",
|
||||
"step": 60
|
||||
}
|
||||
],
|
||||
"title": "Filer Object Write Rate by Size Range",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Objects created per size range over the selected time window.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 131
|
||||
},
|
||||
"id": 94,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"valueMode": "color"
|
||||
},
|
||||
"pluginVersion": "8.1.2",
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range]))",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "< 1KB",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1KB - 100KB",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100KB - 1MB",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "1MB - 100MB",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "100MB - 1GB",
|
||||
"refId": "E"
|
||||
},
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "> 1GB",
|
||||
"refId": "F"
|
||||
}
|
||||
],
|
||||
"title": "Filer Object Size Distribution",
|
||||
"type": "bargauge"
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
|
||||
@@ -15,26 +15,15 @@ metadata:
|
||||
{{- toYaml .Values.admin.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- $serviceType := .Values.admin.service.type }}
|
||||
{{- $nodePorts := dict }}
|
||||
{{- if or (eq $serviceType "NodePort") (eq $serviceType "LoadBalancer") }}
|
||||
{{- $nodePorts = .Values.admin.service.nodePorts | default dict }}
|
||||
{{- end }}
|
||||
type: {{ $serviceType }}
|
||||
type: {{ .Values.admin.service.type }}
|
||||
ports:
|
||||
- name: "http"
|
||||
port: {{ .Values.admin.port }}
|
||||
targetPort: {{ .Values.admin.port }}
|
||||
{{- if $nodePorts.http }}
|
||||
nodePort: {{ $nodePorts.http }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
- name: "grpc"
|
||||
port: {{ .Values.admin.grpcPort }}
|
||||
targetPort: {{ .Values.admin.grpcPort }}
|
||||
{{- if $nodePorts.grpc }}
|
||||
nodePort: {{ $nodePorts.grpc }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user