diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 11b339bc6..1800de685 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -6,6 +6,7 @@ on: paths: - 'weed/**' - 'seaweed-volume/**' + - 'seaweed-worker/**' - 'docker/**' - 'go.mod' - 'go.sum' @@ -16,7 +17,7 @@ permissions: jobs: - # ── Pre-build Rust volume server binaries natively ────────────────── + # ── Pre-build the Rust binaries natively ──────────────────────────── build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -48,6 +49,11 @@ jobs: # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + - name: Cache cargo registry and target uses: actions/cache@v6 with: @@ -55,7 +61,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-dev-${{ matrix.target }}- @@ -67,11 +74,19 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} - path: weed-volume-normal-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} + path: | + weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} build-dev-containers: needs: [build-rust-binaries] @@ -84,7 +99,7 @@ jobs: - name: Download pre-built Rust binaries uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -98,7 +113,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index a82755c2e..8ef7eaf4c 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -59,7 +59,7 @@ jobs: echo "publish=true" >> "$GITHUB_OUTPUT" fi - # ── Pre-build Rust volume server binaries natively ────────────────── + # ── Pre-build the Rust binaries natively ──────────────────────────── build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -93,6 +93,11 @@ jobs: # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + - name: Cache cargo registry and target uses: actions/cache@v6 with: @@ -100,7 +105,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -120,13 +126,20 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} path: | weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} build: needs: [setup, build-rust-binaries] @@ -174,7 +187,7 @@ jobs: - name: Download pre-built Rust binaries uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -188,7 +201,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Docker meta id: docker_meta @@ -286,7 +308,7 @@ jobs: if: needs.setup.outputs.publish != 'true' uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins - name: Place Rust binaries in Docker context for local scan @@ -304,7 +326,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Create BuildKit config for local scan build if: needs.setup.outputs.publish != 'true' run: | diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index f8d9cd21d..7a55bad30 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -42,9 +42,10 @@ concurrency: jobs: - # ── Pre-build Rust volume server binaries natively ────────────────── - # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour - # emulated cargo build into ~15 minutes of native compilation. + # ── Pre-build the Rust binaries natively ──────────────────────────── + # The volume server and the Rust maintenance worker, cross-compiled for + # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into + # ~15 minutes of native compilation. build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -76,6 +77,11 @@ jobs: # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + - name: Cache cargo registry and target uses: actions/cache@v6 with: @@ -83,7 +89,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -103,13 +110,20 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} path: | weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} # One job per (variant, platform) on a native runner, pushed by digest; # the merge job stitches the digests into one multi-arch tag. @@ -155,7 +169,7 @@ jobs: if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -170,7 +184,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/rust-worker-tests.yml b/.github/workflows/rust-worker-tests.yml new file mode 100644 index 000000000..f6bbade3d --- /dev/null +++ b/.github/workflows/rust-worker-tests.yml @@ -0,0 +1,72 @@ +name: "Rust Plugin Worker Tests" + +on: + pull_request: + branches: [ master ] + paths: + - 'seaweed-worker/**' + - 'weed/pb/plugin.proto' + - '.github/workflows/rust-worker-tests.yml' + push: + branches: [ master, main ] + paths: + - 'seaweed-worker/**' + - 'weed/pb/plugin.proto' + - '.github/workflows/rust-worker-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + rust-worker-build: + name: Rust Plugin Worker Build and Unit Tests + runs-on: ubuntu-22.04 + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-worker/target/release + key: rust-worker-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-worker-${{ steps.toolchain.outputs.fingerprint }}- + + # The release profile is what ships, and it is where the release and the + # container builds would otherwise discover a break for the first time. + - name: Build the plugin workers + run: cd seaweed-worker && cargo build --release + + # The tests that need a live gateway skip themselves without one, the way + # the Go integration tests skip without Docker; the lifecycle suite in + # test/s3tables/lifecycle is what runs them against a real cluster. + # Release, so this reuses the build above rather than compiling lance, + # arrow and datafusion a second time in another profile. + - name: Run unit tests + run: cd seaweed-worker && cargo test --release --workspace diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index 14a49e835..9cbd5df52 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -1,4 +1,4 @@ -name: "rust: build versioned volume server binaries" +name: "rust: build versioned binaries" on: push: @@ -113,6 +113,92 @@ jobs: weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5 + # The Rust maintenance worker: Linux only, because it runs beside the cluster + # it maintains rather than on a laptop, and its dependency tree (lance, arrow, + # datafusion) makes every extra target an expensive build. + build-rust-worker-linux: + permissions: + contents: write + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-gnu + asset_suffix: linux_arm64 + cross: true + + steps: + - uses: actions/checkout@v7 + with: + # The upload step is handed a token explicitly; a cargo build script + # should not find another one sitting in the checkout's git config. + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross-compilation tools + if: matrix.cross + run: | + sudo dpkg --add-architecture arm64 + sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-worker/target/${{ matrix.target }}/release + key: rust-worker-release-${{ matrix.target }}-${{ hashFiles('seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-worker-release-${{ matrix.target }}- + + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + + - name: Package binary + run: | + cp seaweed-worker/target/${{ matrix.target }}/release/weed-worker weed-worker + tar czf weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker + rm weed-worker + md5sum weed-worker_${{ matrix.asset_suffix }}.tar.gz > weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-worker_${{ matrix.asset_suffix }}.tar.gz + weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-worker-${{ matrix.asset_suffix }} + path: | + weed-worker_${{ matrix.asset_suffix }}.tar.gz + weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + build-rust-volume-darwin: permissions: contents: write diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index 6ca519257..227f58c34 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -29,6 +29,7 @@ FROM alpine:3.23 as rust_builder ARG TARGETARCH ARG TAGS COPY weed-volume-prebuilt/ /prebuilt/ +COPY weed-worker-prebuilt/ /prebuilt-worker/ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed WORKDIR /build/seaweed-volume @@ -47,16 +48,30 @@ RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \ echo "Skipping Rust build for $TARGETARCH (unsupported)" && \ touch /weed-volume; \ fi +# The Rust maintenance worker is taken pre-built or not at all: the lance jobs +# it carries pull in arrow and datafusion, a far larger dependency tree than +# the image build can carry, so an architecture CI did not build for gets the +# same empty placeholder the entrypoint refuses to exec. +RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \ + echo "Using pre-built Rust worker for ${TARGETARCH}" && \ + cp "/prebuilt-worker/weed-worker-${TARGETARCH}" /weed-worker; \ + else \ + echo "No pre-built Rust worker for ${TARGETARCH}" && \ + touch /weed-worker; \ + fi + # Pre-built binaries arrive via GitHub Actions artifacts, which drop the # executable bit, so the copied file is 0644 and exec fails with "Permission -# denied". Restore it (no-op for the empty placeholder, which stays size 0). -RUN chmod 0755 /weed-volume +# denied". Restore it (no-op for the empty placeholders, which stay size 0). +RUN chmod 0755 /weed-volume /weed-worker FROM alpine AS final LABEL author="Chris Lu" COPY --from=builder /go/bin/weed /usr/bin/ # Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms) COPY --from=rust_builder /weed-volume /usr/bin/weed-volume +# Same for the Rust maintenance worker, which serves Lance table buckets +COPY --from=rust_builder /weed-worker /usr/bin/weed-worker RUN mkdir -p /etc/seaweedfs COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7ce0595ec..0bbd50787 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -90,6 +90,16 @@ case "$1" in exec /usr/bin/weed-volume $ARGS $@ ;; + 'worker-rust') + shift + if [ ! -s /usr/bin/weed-worker ]; then + echo "Error: Rust maintenance worker is not available on this platform ($(uname -m))." >&2 + echo "Use 'worker' for the Go maintenance worker instead." >&2 + exit 1 + fi + exec /usr/bin/weed-worker "$@" + ;; + 'server') ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024" if isArgPassed "-volume.max" "$@"; then diff --git a/docker/weed-worker-prebuilt/.gitkeep b/docker/weed-worker-prebuilt/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md index e43b905fe..080ddfd4c 100644 --- a/seaweed-worker/README.md +++ b/seaweed-worker/README.md @@ -20,6 +20,22 @@ The admin's *HTTP* address is what an operator has; the gRPC port is derived from it the way the Go side does. Dialling the HTTP port fails as "frame with invalid size", which reads like a protocol bug rather than a wrong port. +The binary is `weed-worker`, not `weed-lance-worker`: it is the Rust side of +`weed worker`, and lance is the first family of jobs it carries rather than the +only one it ever will. + +Released builds do not need a toolchain. The worker ships inside the SeaweedFS +image, beside the Rust volume server, under the verb that mirrors +`volume-rust`: + + docker run chrislusf/seaweedfs worker-rust --admin admin:23646 + +and as `weed-worker_linux_{amd64,arm64}.tar.gz` on each GitHub release. Both are +linux amd64/arm64 only — lance, arrow and datafusion make every extra target an +expensive build, and the worker runs beside the cluster it maintains. On an +architecture without a build the image carries an empty placeholder and the +entrypoint says so rather than failing as "not found". + ## Metrics cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646 --metrics-port 9328 diff --git a/seaweed-worker/crates/lance/Cargo.toml b/seaweed-worker/crates/lance/Cargo.toml index ab59e1e6c..e431e7c96 100644 --- a/seaweed-worker/crates/lance/Cargo.toml +++ b/seaweed-worker/crates/lance/Cargo.toml @@ -7,8 +7,11 @@ description = "SeaweedFS maintenance worker for Lance tables" [lib] name = "weed_lance_worker" +# The binary is not named for lance: it is the Rust side of `weed worker`, and +# the job families it registers will outgrow this crate. When a second one +# arrives the bin target moves to a crate of its own under the same name. [[bin]] -name = "weed-lance-worker" +name = "weed-worker" path = "src/main.rs" [dependencies] diff --git a/seaweed-worker/crates/lance/src/main.rs b/seaweed-worker/crates/lance/src/main.rs index 41bb1109b..cd7015a92 100644 --- a/seaweed-worker/crates/lance/src/main.rs +++ b/seaweed-worker/crates/lance/src/main.rs @@ -13,8 +13,8 @@ use weed_lance_worker::metrics::LanceMetrics; /// language and an operator should not have to learn a second set of names. #[derive(Parser, Debug)] #[command( - name = "weed-lance-worker", - about = "SeaweedFS maintenance worker for Lance tables" + name = "weed-worker", + about = "SeaweedFS maintenance worker" )] struct Args { /// Admin server gRPC address.