From 3f68b1950028f463d24c19afe1a0d20a2664b933 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 25 Jun 2026 00:37:33 -0700 Subject: [PATCH] docker release: per-platform builds on native runners, drop mode=max cache (#10109) docker release: build per-platform on native runners, drop mode=max cache The build job built every platform of a variant on one runner, so 2-4 Go cross-compiles fought over a single 2-vCPU box and arm64 ran in an emulated context. Split the matrix to one platform per job on a native runner (amd64/386 on ubuntu-latest, arm64/arm-v7 on ubuntu-24.04-arm); only arm/v7 still needs QEMU, and only for its final apk stage. Each job pushes by digest, and a new merge job assembles the multi-arch tag with imagetools and mirrors it to Docker Hub. cache-to mode=max -> mode=min: BRANCH=sha cache-busts the heavy go-build layer every release, so writing all intermediate layers to the gha backend spent 3-11 min per variant on a cache the next release's sha can never hit. --- .../workflows/container_release_unified.yml | 210 +++++++++++------- 1 file changed, 127 insertions(+), 83 deletions(-) diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 816e61d3a..e4ac39c28 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -33,6 +33,7 @@ permissions: env: RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }} + IMAGE: ghcr.io/chrislusf/seaweedfs # Limit concurrent builds to avoid rate limits concurrency: @@ -113,57 +114,51 @@ jobs: weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} - # ── Build Docker containers ───────────────────────────────────────── + # ── Build per-platform images on native runners ───────────────────── + # Each (variant, platform) builds on a single runner whose CPU matches + # the target arch, then pushes by digest (no tag). The merge job below + # assembles the per-platform digests into one multi-arch tag. Splitting + # by platform parallelises the Go cross-compiles across machines instead + # of contending for one 2-vCPU runner, and lets amd64/arm64 build native. + # + # cache-to is mode=min (not max): BRANCH=github.sha cache-busts the heavy + # go-build layer on every release, so caching intermediate stages just + # paid minutes to write a cache that the next release's SHA can't hit. build: needs: [build-rust-binaries] - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} strategy: - # All variants at once: pulls hit mirror.gcr.io and pushes go to GHCR only, - # so docker.io limits don't apply. Watch the 10 GB gha cache budget. - max-parallel: 5 + fail-fast: false + max-parallel: 8 matrix: include: # Normal volume - multi-arch - - variant: normal - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386 - dockerfile: ./docker/Dockerfile.go_build - build_args: "" - tag_suffix: "" - rust_variant: normal + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true } + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false } # Large disk - multi-arch - - variant: large_disk - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386 - dockerfile: ./docker/Dockerfile.go_build - build_args: TAGS=5BytesOffset - tag_suffix: _large_disk - rust_variant: large-disk + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true } + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false } # Full tags - multi-arch - - variant: full - platforms: linux/amd64,linux/arm64 - dockerfile: ./docker/Dockerfile.go_build - build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb - tag_suffix: _full - rust_variant: normal + - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } # Large disk + full tags - multi-arch - - variant: large_disk_full - platforms: linux/amd64,linux/arm64 - dockerfile: ./docker/Dockerfile.go_build - build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb - tag_suffix: _large_disk_full - rust_variant: large-disk + - { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } # RocksDB large disk - amd64 only - - variant: rocksdb - platforms: linux/amd64 - dockerfile: ./docker/Dockerfile.rocksdb_large - build_args: "" - tag_suffix: _large_disk_rocksdb - rust_variant: large-disk - + - { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } steps: + - name: Skip unselected variant + if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant + run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0 + - name: Checkout if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/checkout@v7 @@ -192,36 +187,31 @@ jobs: - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant run: | - echo "Available disk space before cleanup:" - df -h sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL sudo apt-get clean sudo rm -rf /var/lib/apt/lists/* sudo docker system prune -af --volumes [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true [ -d /go/pkg ] && rm -rf /go/pkg || true - echo "Available disk space after cleanup:" df -h - + - name: Docker meta if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant id: docker_meta uses: docker/metadata-action@v6 with: - images: | - chrislusf/seaweedfs - ghcr.io/chrislusf/seaweedfs + images: ${{ env.IMAGE }} tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} flavor: latest=false labels: | org.opencontainers.image.title=seaweedfs org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! org.opencontainers.image.vendor=Chris Lu - + - name: Set up QEMU - if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm') + if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu uses: docker/setup-qemu-action@v4.1.0 - + - name: Create BuildKit config if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant run: | @@ -229,54 +219,114 @@ jobs: [registry."docker.io"] mirrors = ["https://mirror.gcr.io"] EOF - + - name: Set up Docker Buildx if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: docker/setup-buildx-action@v4 with: buildkitd-config: /tmp/buildkitd.toml - - - name: Login to Docker Hub - if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request' - uses: docker/login-action@v4.2.0 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - + - name: Login to GHCR - if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request' + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: docker/login-action@v4.2.0 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - - - name: Build and push ${{ matrix.variant }} + + - name: Build and push ${{ matrix.variant }} (${{ matrix.platform }}) if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + id: build uses: docker/build-push-action@v7 env: DOCKER_BUILDKIT: 1 with: context: ./docker - push: ${{ github.event_name != 'pull_request' }} file: ${{ matrix.dockerfile }} - platforms: ${{ matrix.platforms }} - # Push to GHCR to avoid Docker Hub rate limits on pulls - tags: | - ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} + platforms: ${{ matrix.platform }} labels: ${{ steps.docker_meta.outputs.labels }} - cache-from: type=gha,scope=${{ matrix.variant }} - cache-to: type=gha,mode=max,scope=${{ matrix.variant }} + # Single manifest per platform so imagetools create builds a flat index. + provenance: false + # Push by digest (no tag); the merge job assembles the multi-arch tag. + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }} + cache-to: type=gha,mode=min,scope=${{ matrix.variant }}-${{ matrix.arch }} build-args: | ${{ matrix.build_args }} BUILDKIT_INLINE_CACHE=1 BRANCH=${{ github.sha }} ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }} - - # Copy GHCR -> Docker Hub here, per variant, so it overlaps with the other - # variants still building instead of waiting on the whole matrix. + + - name: Export digest + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: actions/upload-artifact@v7 + with: + name: digest-${{ matrix.variant }}-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # ── Assemble per-variant multi-arch tag, then mirror to Docker Hub ─── + # imagetools create stitches the per-platform digests into one tag — no + # rebuild. crane copy then mirrors the whole index GHCR -> Docker Hub. + merge: + needs: [build] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { variant: normal, tag_suffix: "" } + - { variant: large_disk, tag_suffix: _large_disk } + - { variant: full, tag_suffix: _full } + - { variant: large_disk_full, tag_suffix: _large_disk_full } + - { variant: rocksdb, tag_suffix: _large_disk_rocksdb } + steps: + - name: Download digests + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: actions/download-artifact@v8 + with: + pattern: digest-${{ matrix.variant }}-* + merge-multiple: true + path: /tmp/digests + + - name: Set up Docker Buildx + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: docker/setup-buildx-action@v4 + + - name: Login to GHCR + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: docker/login-action@v4.2.0 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Create multi-arch tag on GHCR + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + working-directory: /tmp/digests + run: | + docker buildx imagetools create \ + -t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \ + $(printf '${{ env.IMAGE }}@sha256:%s ' *) + docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} + + - name: Login to Docker Hub + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: docker/login-action@v4.2.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Install crane - if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request' + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant run: | cd $(mktemp -d) curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz @@ -284,7 +334,7 @@ jobs: crane version - name: Copy ${{ matrix.variant }} to Docker Hub - if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request' + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant run: | retry_with_backoff() { local max_attempts=5 @@ -310,16 +360,10 @@ jobs: echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..." retry_with_backoff crane copy \ - ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \ + ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \ chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} echo "Copied ${{ matrix.variant }} to Docker Hub" - - name: Clean up build artifacts - if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) - run: | - sudo docker system prune -f - sudo rm -rf /tmp/go-build* - # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub # Security for visibility, but never blocks the release. Releases (including # `latest`) ship regardless — vulnerabilities are tracked, not gated, since @@ -327,7 +371,7 @@ jobs: # CVE, base-image lag). trivy-scan: runs-on: ubuntu-latest - needs: [build] + needs: [merge] if: github.event_name == 'push' continue-on-error: true strategy: @@ -354,7 +398,7 @@ jobs: # Scan the multi-arch tag on GHCR (already pushed by the build job). # Trivy scans the runner's native platform; OS packages are identical # across architectures since they all share the same alpine base. - image-ref: ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} + image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} scanners: vuln vuln-type: os,library severity: HIGH,CRITICAL @@ -378,7 +422,7 @@ jobs: # trivy-scan: vuln findings are reported but do not block `latest`. tag-latest: runs-on: ubuntu-latest - needs: [build] + needs: [merge] if: github.event_name == 'push' strategy: matrix: @@ -435,8 +479,8 @@ jobs: SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}" DST_TAG="${{ matrix.latest_tag }}" - echo "Tagging ghcr.io/chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}" - retry_with_backoff crane tag "ghcr.io/chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}" + echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}" + retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}" echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}" retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}" @@ -452,7 +496,7 @@ jobs: - uses: actions/checkout@v7 - name: Publish Helm charts uses: stefanprodan/helm-gh-pages@v1.7.0 - with: + with: token: ${{ secrets.GITHUB_TOKEN }} charts_dir: k8s/charts target_dir: helm