From cc8364a03ebf5cf9f8eab3315ded2000cd07fff6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 22 Aug 2026 10:22:33 -0700 Subject: [PATCH] Ship the Rust maintenance worker with the release (#10879) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * worker: name the binary weed-worker It is the Rust side of `weed worker`, the way weed-volume is the Rust side of `weed volume`, and lance is the first family of jobs it carries rather than the only one it ever will. The crate keeps its own name: when a second family arrives the bin target moves to a crate of its own, under this name. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * docker: ship the Rust maintenance worker in the image Lance table buckets need a worker that can read the format, and until now the only way to get one was a Rust toolchain and a cargo build. It now sits at /usr/bin/weed-worker beside the Rust volume server, reached as `docker run chrislusf/seaweedfs worker-rust --admin host:23646` — the verb mirrors volume-rust, so plain `worker` still runs the Go one. Taken pre-built or not at all: the lance jobs pull in arrow and datafusion, far too large a tree to compile inside the image build, so an architecture CI did not build for gets the empty placeholder the entrypoint refuses to exec, the way the Rust volume server already does. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * ci: build the Rust worker for the container images The same native cross-compile the volume server uses, so the release, latest and dev images all carry it on amd64 and arm64. The artifact holds both binaries now, so it is named for that rather than for the volume server. Only the release directory each job builds is cached: with a debug profile beside it the worker's target/ reaches 24GB, against a 10GB cache budget. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * ci: publish Rust worker binaries with the release Linux amd64 and arm64 only: the worker runs beside the cluster it maintains, and its dependency tree makes every extra target an expensive build. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * ci: build and test the Rust workers on change Nothing built seaweed-worker in CI, so the release and the container images would have been the first place a break showed up. Tests run in release too, rather than compiling lance, arrow and datafusion again in another profile. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * docs: say how to get a released worker Neither the image nor the release tarballs were mentioned; a toolchain and a cargo build read as the only way in. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * ci: install protoc for the Rust worker builds lance's crates compile their own protos, and unlike seaweed-volume they do not vendor a protoc to do it with, so every job that builds the worker failed at lance-encoding's build script. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * ci: do not persist credentials in the worker release checkout The upload step is handed a token explicitly; a cargo build script should not find another one sitting in the checkout's git config. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm * docker: keep the worker's argument boundaries Unquoted $@ splits on whitespace and expands globs, so an argument carrying either arrived as something else. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm --- .github/workflows/container_dev.yml | 34 +++++-- .github/workflows/container_latest.yml | 41 +++++++-- .../workflows/container_release_unified.yml | 35 ++++++-- .github/workflows/rust-worker-tests.yml | 72 +++++++++++++++ .github/workflows/rust_binaries_release.yml | 88 ++++++++++++++++++- docker/Dockerfile.go_build | 19 +++- docker/entrypoint.sh | 10 +++ docker/weed-worker-prebuilt/.gitkeep | 0 seaweed-worker/README.md | 16 ++++ seaweed-worker/crates/lance/Cargo.toml | 5 +- seaweed-worker/crates/lance/src/main.rs | 4 +- 11 files changed, 302 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/rust-worker-tests.yml create mode 100644 docker/weed-worker-prebuilt/.gitkeep diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 11b339bc6..1800de685 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -6,6 +6,7 @@ on: paths: - 'weed/**' - 'seaweed-volume/**' + - 'seaweed-worker/**' - 'docker/**' - 'go.mod' - 'go.sum' @@ -16,7 +17,7 @@ permissions: jobs: - # ── Pre-build Rust volume server binaries natively ────────────────── + # ── Pre-build the Rust binaries natively ──────────────────────────── build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -48,6 +49,11 @@ jobs: # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + - name: Cache cargo registry and target uses: actions/cache@v6 with: @@ -55,7 +61,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-dev-${{ matrix.target }}- @@ -67,11 +74,19 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} - path: weed-volume-normal-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} + path: | + weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} build-dev-containers: needs: [build-rust-binaries] @@ -84,7 +99,7 @@ jobs: - name: Download pre-built Rust binaries uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -98,7 +113,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index a82755c2e..8ef7eaf4c 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -59,7 +59,7 @@ jobs: echo "publish=true" >> "$GITHUB_OUTPUT" fi - # ── Pre-build Rust volume server binaries natively ────────────────── + # ── Pre-build the Rust binaries natively ──────────────────────────── build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -93,6 +93,11 @@ jobs: # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + - name: Cache cargo registry and target uses: actions/cache@v6 with: @@ -100,7 +105,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -120,13 +126,20 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} path: | weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} build: needs: [setup, build-rust-binaries] @@ -174,7 +187,7 @@ jobs: - name: Download pre-built Rust binaries uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -188,7 +201,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Docker meta id: docker_meta @@ -286,7 +308,7 @@ jobs: if: needs.setup.outputs.publish != 'true' uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins - name: Place Rust binaries in Docker context for local scan @@ -304,7 +326,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Create BuildKit config for local scan build if: needs.setup.outputs.publish != 'true' run: | diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index f8d9cd21d..7a55bad30 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -42,9 +42,10 @@ concurrency: jobs: - # ── Pre-build Rust volume server binaries natively ────────────────── - # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour - # emulated cargo build into ~15 minutes of native compilation. + # ── Pre-build the Rust binaries natively ──────────────────────────── + # The volume server and the Rust maintenance worker, cross-compiled for + # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into + # ~15 minutes of native compilation. build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -76,6 +77,11 @@ jobs: # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + - name: Cache cargo registry and target uses: actions/cache@v6 with: @@ -83,7 +89,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -103,13 +110,20 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} path: | weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} # One job per (variant, platform) on a native runner, pushed by digest; # the merge job stitches the digests into one multi-arch tag. @@ -155,7 +169,7 @@ jobs: if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -170,7 +184,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/rust-worker-tests.yml b/.github/workflows/rust-worker-tests.yml new file mode 100644 index 000000000..f6bbade3d --- /dev/null +++ b/.github/workflows/rust-worker-tests.yml @@ -0,0 +1,72 @@ +name: "Rust Plugin Worker Tests" + +on: + pull_request: + branches: [ master ] + paths: + - 'seaweed-worker/**' + - 'weed/pb/plugin.proto' + - '.github/workflows/rust-worker-tests.yml' + push: + branches: [ master, main ] + paths: + - 'seaweed-worker/**' + - 'weed/pb/plugin.proto' + - '.github/workflows/rust-worker-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + rust-worker-build: + name: Rust Plugin Worker Build and Unit Tests + runs-on: ubuntu-22.04 + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-worker/target/release + key: rust-worker-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-worker-${{ steps.toolchain.outputs.fingerprint }}- + + # The release profile is what ships, and it is where the release and the + # container builds would otherwise discover a break for the first time. + - name: Build the plugin workers + run: cd seaweed-worker && cargo build --release + + # The tests that need a live gateway skip themselves without one, the way + # the Go integration tests skip without Docker; the lifecycle suite in + # test/s3tables/lifecycle is what runs them against a real cluster. + # Release, so this reuses the build above rather than compiling lance, + # arrow and datafusion a second time in another profile. + - name: Run unit tests + run: cd seaweed-worker && cargo test --release --workspace diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index 14a49e835..9cbd5df52 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -1,4 +1,4 @@ -name: "rust: build versioned volume server binaries" +name: "rust: build versioned binaries" on: push: @@ -113,6 +113,92 @@ jobs: weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5 + # The Rust maintenance worker: Linux only, because it runs beside the cluster + # it maintains rather than on a laptop, and its dependency tree (lance, arrow, + # datafusion) makes every extra target an expensive build. + build-rust-worker-linux: + permissions: + contents: write + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-gnu + asset_suffix: linux_arm64 + cross: true + + steps: + - uses: actions/checkout@v7 + with: + # The upload step is handed a token explicitly; a cargo build script + # should not find another one sitting in the checkout's git config. + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross-compilation tools + if: matrix.cross + run: | + sudo dpkg --add-architecture arm64 + sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + + # lance's build scripts compile their own protos, and unlike seaweed-volume + # they do not vendor a protoc to do it with. + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-worker/target/${{ matrix.target }}/release + key: rust-worker-release-${{ matrix.target }}-${{ hashFiles('seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-worker-release-${{ matrix.target }}- + + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + + - name: Package binary + run: | + cp seaweed-worker/target/${{ matrix.target }}/release/weed-worker weed-worker + tar czf weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker + rm weed-worker + md5sum weed-worker_${{ matrix.asset_suffix }}.tar.gz > weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-worker_${{ matrix.asset_suffix }}.tar.gz + weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-worker-${{ matrix.asset_suffix }} + path: | + weed-worker_${{ matrix.asset_suffix }}.tar.gz + weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + build-rust-volume-darwin: permissions: contents: write diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index 6ca519257..227f58c34 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -29,6 +29,7 @@ FROM alpine:3.23 as rust_builder ARG TARGETARCH ARG TAGS COPY weed-volume-prebuilt/ /prebuilt/ +COPY weed-worker-prebuilt/ /prebuilt-worker/ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed WORKDIR /build/seaweed-volume @@ -47,16 +48,30 @@ RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \ echo "Skipping Rust build for $TARGETARCH (unsupported)" && \ touch /weed-volume; \ fi +# The Rust maintenance worker is taken pre-built or not at all: the lance jobs +# it carries pull in arrow and datafusion, a far larger dependency tree than +# the image build can carry, so an architecture CI did not build for gets the +# same empty placeholder the entrypoint refuses to exec. +RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \ + echo "Using pre-built Rust worker for ${TARGETARCH}" && \ + cp "/prebuilt-worker/weed-worker-${TARGETARCH}" /weed-worker; \ + else \ + echo "No pre-built Rust worker for ${TARGETARCH}" && \ + touch /weed-worker; \ + fi + # Pre-built binaries arrive via GitHub Actions artifacts, which drop the # executable bit, so the copied file is 0644 and exec fails with "Permission -# denied". Restore it (no-op for the empty placeholder, which stays size 0). -RUN chmod 0755 /weed-volume +# denied". Restore it (no-op for the empty placeholders, which stay size 0). +RUN chmod 0755 /weed-volume /weed-worker FROM alpine AS final LABEL author="Chris Lu" COPY --from=builder /go/bin/weed /usr/bin/ # Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms) COPY --from=rust_builder /weed-volume /usr/bin/weed-volume +# Same for the Rust maintenance worker, which serves Lance table buckets +COPY --from=rust_builder /weed-worker /usr/bin/weed-worker RUN mkdir -p /etc/seaweedfs COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7ce0595ec..0bbd50787 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -90,6 +90,16 @@ case "$1" in exec /usr/bin/weed-volume $ARGS $@ ;; + 'worker-rust') + shift + if [ ! -s /usr/bin/weed-worker ]; then + echo "Error: Rust maintenance worker is not available on this platform ($(uname -m))." >&2 + echo "Use 'worker' for the Go maintenance worker instead." >&2 + exit 1 + fi + exec /usr/bin/weed-worker "$@" + ;; + 'server') ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024" if isArgPassed "-volume.max" "$@"; then diff --git a/docker/weed-worker-prebuilt/.gitkeep b/docker/weed-worker-prebuilt/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md index e43b905fe..080ddfd4c 100644 --- a/seaweed-worker/README.md +++ b/seaweed-worker/README.md @@ -20,6 +20,22 @@ The admin's *HTTP* address is what an operator has; the gRPC port is derived from it the way the Go side does. Dialling the HTTP port fails as "frame with invalid size", which reads like a protocol bug rather than a wrong port. +The binary is `weed-worker`, not `weed-lance-worker`: it is the Rust side of +`weed worker`, and lance is the first family of jobs it carries rather than the +only one it ever will. + +Released builds do not need a toolchain. The worker ships inside the SeaweedFS +image, beside the Rust volume server, under the verb that mirrors +`volume-rust`: + + docker run chrislusf/seaweedfs worker-rust --admin admin:23646 + +and as `weed-worker_linux_{amd64,arm64}.tar.gz` on each GitHub release. Both are +linux amd64/arm64 only — lance, arrow and datafusion make every extra target an +expensive build, and the worker runs beside the cluster it maintains. On an +architecture without a build the image carries an empty placeholder and the +entrypoint says so rather than failing as "not found". + ## Metrics cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646 --metrics-port 9328 diff --git a/seaweed-worker/crates/lance/Cargo.toml b/seaweed-worker/crates/lance/Cargo.toml index ab59e1e6c..e431e7c96 100644 --- a/seaweed-worker/crates/lance/Cargo.toml +++ b/seaweed-worker/crates/lance/Cargo.toml @@ -7,8 +7,11 @@ description = "SeaweedFS maintenance worker for Lance tables" [lib] name = "weed_lance_worker" +# The binary is not named for lance: it is the Rust side of `weed worker`, and +# the job families it registers will outgrow this crate. When a second one +# arrives the bin target moves to a crate of its own under the same name. [[bin]] -name = "weed-lance-worker" +name = "weed-worker" path = "src/main.rs" [dependencies] diff --git a/seaweed-worker/crates/lance/src/main.rs b/seaweed-worker/crates/lance/src/main.rs index 41bb1109b..cd7015a92 100644 --- a/seaweed-worker/crates/lance/src/main.rs +++ b/seaweed-worker/crates/lance/src/main.rs @@ -13,8 +13,8 @@ use weed_lance_worker::metrics::LanceMetrics; /// language and an operator should not have to learn a second set of names. #[derive(Parser, Debug)] #[command( - name = "weed-lance-worker", - about = "SeaweedFS maintenance worker for Lance tables" + name = "weed-worker", + about = "SeaweedFS maintenance worker" )] struct Args { /// Admin server gRPC address.