From baead6901c581c3a28e01d00d3375ddaa7a92b0a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 19 Aug 2026 00:18:30 -0700 Subject: [PATCH] ci: build protoc into the crate instead of installing it per job (#10830) Every workflow that builds the Rust volume server first installed protoc from a package manager - twelve steps across apt, brew and choco. That is 37s per job on a good day, and this week archive.ubuntu.com stalled long enough for four jobs to burn their whole timeout without reaching a build. protoc-bin-vendored ships the compiler as a build-dependency, so it now arrives through the cargo registry the workflows already cache and there is nothing left to install. cargo build works on a machine with no protoc at all, which is worth as much locally as it is in CI. It also pins the version. The apt protoc on ubuntu-22.04 is 3.12, old enough to reject proto3 optional, which is why build.rs passes --experimental_allow_proto3_optional; the vendored one is 31.1. The flag stays, since it costs nothing and keeps a build against an older PROTOC working, and an explicit PROTOC still overrides the vendored binary for packagers who supply their own. --- .github/workflows/container_dev.yml | 3 - .github/workflows/container_latest.yml | 3 - .../workflows/container_release_unified.yml | 3 - .github/workflows/performance.yml | 8 --- .../workflows/rust-volume-server-tests.yml | 9 --- .github/workflows/rust_binaries_dev.yml | 6 -- .github/workflows/rust_binaries_release.yml | 9 --- seaweed-volume/Cargo.lock | 65 +++++++++++++++++++ seaweed-volume/Cargo.toml | 4 ++ seaweed-volume/build.rs | 12 +++- 10 files changed, 79 insertions(+), 43 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 8d72fa4a7..11b339bc6 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -31,9 +31,6 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 23193e573..a82755c2e 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -76,9 +76,6 @@ jobs: with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 198de19ef..f8d9cd21d 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -59,9 +59,6 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index cd5c521da..2be3782bd 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -175,10 +175,6 @@ jobs: with: go-version-file: 'go.mod' - - name: Install protobuf compiler - if: matrix.impl == 'rust' - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain if: matrix.impl == 'rust' uses: dtolnay/rust-toolchain@stable @@ -331,10 +327,6 @@ jobs: with: go-version-file: 'go.mod' - - name: Install protobuf compiler - if: matrix.impl == 'rust' - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain if: matrix.impl == 'rust' uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index 6f2417296..ba8e09a39 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -36,9 +36,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -79,9 +76,6 @@ jobs: with: go-version-file: 'go.mod' - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -164,9 +158,6 @@ jobs: with: go-version-file: 'go.mod' - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/rust_binaries_dev.yml b/.github/workflows/rust_binaries_dev.yml index 9c4aacc71..c8cb4f0c6 100644 --- a/.github/workflows/rust_binaries_dev.yml +++ b/.github/workflows/rust_binaries_dev.yml @@ -41,9 +41,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -110,9 +107,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install protobuf compiler - run: brew install protobuf - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index 46c596fc3..14a49e835 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -28,9 +28,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -131,9 +128,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install protobuf compiler - run: brew install protobuf - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -211,9 +205,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Install protobuf compiler - run: choco install protoc -y - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable diff --git a/seaweed-volume/Cargo.lock b/seaweed-volume/Cargo.lock index a4c875258..b0ce04703 100644 --- a/seaweed-volume/Cargo.lock +++ b/seaweed-volume/Cargo.lock @@ -2929,6 +2929,70 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "pxfm" version = "0.1.28" @@ -4529,6 +4593,7 @@ dependencies = [ "prometheus", "prost 0.13.5", "prost-types 0.13.5", + "protoc-bin-vendored", "rand 0.10.2", "redb", "reed-solomon-erasure", diff --git a/seaweed-volume/Cargo.toml b/seaweed-volume/Cargo.toml index 481f7bc12..91db873ea 100644 --- a/seaweed-volume/Cargo.toml +++ b/seaweed-volume/Cargo.toml @@ -137,6 +137,10 @@ tempfile = "3" [build-dependencies] tonic-build = "0.12" +# Ships protoc with the build so neither CI nor a developer needs a system +# install, and so the version is pinned rather than whatever the platform's +# package manager happens to carry. +protoc-bin-vendored = "3" [patch.crates-io] reed-solomon-erasure = { path = "vendor/reed-solomon-erasure" } diff --git a/seaweed-volume/build.rs b/seaweed-volume/build.rs index a42b8bdfc..99af370a3 100644 --- a/seaweed-volume/build.rs +++ b/seaweed-volume/build.rs @@ -1,10 +1,18 @@ fn main() -> Result<(), Box> { + // Use the protoc that ships with protoc-bin-vendored rather than a system + // one, so the build needs no package manager and always sees the same + // version. An explicit PROTOC still wins, for packagers supplying their own. + if std::env::var_os("PROTOC").is_none() { + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?); + } + let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?); tonic_build::configure() .build_server(true) .build_client(true) - // filer.proto uses proto3 optional; older protoc (e.g. 3.12 from ubuntu-22.04 apt) - // rejects it without this flag, and newer protoc still accepts the flag + // filer.proto uses proto3 optional, which protoc rejects without this + // flag before 3.15. The vendored protoc is newer, but it still accepts + // the flag, so this keeps a build against an older PROTOC working. .protoc_arg("--experimental_allow_proto3_optional") .file_descriptor_set_path(out_dir.join("seaweed_descriptor.bin")) .compile_protos(