From 0c2576c3d0e84cca5d6da33fdafa8422bea72748 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 9 Jun 2026 17:12:42 -0700 Subject: [PATCH] ci: route Docker Hub pulls through a mirror to cut registry timeouts (#9904) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(s3tables): route Docker Hub pulls through mirror, drop unused buildx The integration jobs set up docker/setup-buildx-action only to docker pull/run images; the buildx bootstrap pulls moby/buildkit from registry-1.docker.io, which times out and fails the whole job before any test runs. These jobs never docker build with buildx, so the setup is pure overhead and an extra registry dependency. Replace it with a daemon registry-mirror pointing at mirror.gcr.io (a pull-through cache for Docker Hub) and retry the pre-pulls a few times. That removes the buildkit pull entirely and routes the rest through the cache, with graceful fallback to Docker Hub on a miss. * ci: route Docker Hub through mirror in remaining docker test workflows Same registry-1.docker.io timeout fix for the other integration jobs. s3-spark only docker pulls/runs an image, so drop the vestigial buildx setup and pull through the mirror with retries, matching s3-tables. kafka-quicktest, s3-proxy-signature, e2e and postgres build/compose and genuinely need buildx (e2e/postgres export a local layer cache, which the default driver can't), so keep it and just configure the mirror first — that way even the moby/buildkit bootstrap pull is served from the cache. Left samba/pjdfstest alone: they build-push to a local registry and pull from localhost, so buildx is required and there's no Docker Hub runtime pull to mirror. --- .github/workflows/e2e.yml | 5 + .github/workflows/kafka-quicktest.yml | 5 + .github/workflows/postgres-tests.yml | 5 + .../workflows/s3-proxy-signature-tests.yml | 5 + .github/workflows/s3-spark-tests.yml | 10 +- .github/workflows/s3-tables-tests.yml | 120 ++++++++++++------ 6 files changed, 105 insertions(+), 45 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 433e99c26..cd20839aa 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,6 +31,11 @@ jobs: with: go-version-file: 'go.mod' + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/kafka-quicktest.yml b/.github/workflows/kafka-quicktest.yml index ba696d800..441154984 100644 --- a/.github/workflows/kafka-quicktest.yml +++ b/.github/workflows/kafka-quicktest.yml @@ -32,6 +32,11 @@ jobs: **/go.sum id: go + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/postgres-tests.yml b/.github/workflows/postgres-tests.yml index 5009162e7..d8fd1272e 100644 --- a/.github/workflows/postgres-tests.yml +++ b/.github/workflows/postgres-tests.yml @@ -31,6 +31,11 @@ jobs: - name: Check out code uses: actions/checkout@v6 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/s3-proxy-signature-tests.yml b/.github/workflows/s3-proxy-signature-tests.yml index 34892f333..73da34702 100644 --- a/.github/workflows/s3-proxy-signature-tests.yml +++ b/.github/workflows/s3-proxy-signature-tests.yml @@ -28,6 +28,11 @@ jobs: go-version-file: 'go.mod' id: go + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/s3-spark-tests.yml b/.github/workflows/s3-spark-tests.yml index 106ee93c5..8154a323b 100644 --- a/.github/workflows/s3-spark-tests.yml +++ b/.github/workflows/s3-spark-tests.yml @@ -33,15 +33,19 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Pre-pull Spark image - run: docker pull apache/spark:3.5.8 + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/spark:3.5.8 - name: Run S3 Spark integration tests working-directory: test/s3/spark diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index 3bd245427..bbc28e412 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -83,6 +83,11 @@ jobs: go-version-file: 'go.mod' id: go + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Run go mod tidy run: go mod tidy @@ -139,11 +144,15 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - name: Pre-pull Trino image - run: docker pull trinodb/trino:479 + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull trinodb/trino:479 - name: Run go mod tidy run: go mod tidy @@ -205,14 +214,16 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - - name: Pre-pull Dremio image - run: docker pull dremio/dremio-oss:25.2.0 - - - name: Pre-pull Python image for PyIceberg writer - run: docker pull python:3.11-slim + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull dremio/dremio-oss:25.2.0 + pull python:3.11-slim - name: Run go mod tidy run: go mod tidy @@ -276,14 +287,16 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - - name: Pre-pull Doris image - run: docker pull apache/doris:doris-all-in-one-2.1.0 - - - name: Pre-pull Python image for PyIceberg writer - run: docker pull python:3.11-slim + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/doris:doris-all-in-one-2.1.0 + pull python:3.11-slim - name: Run go mod tidy run: go mod tidy @@ -354,8 +367,15 @@ jobs: run: | go install -buildvcs=false ./weed + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Pre-pull Polaris image - run: docker pull apache/polaris:latest + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/polaris:latest - name: Run Polaris Integration Tests timeout-minutes: 25 @@ -408,11 +428,15 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - name: Pre-pull Spark image - run: docker pull apache/spark:3.5.1 + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/spark:3.5.1 - name: Run go mod tidy run: go mod tidy @@ -474,13 +498,16 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - name: Pre-pull RisingWave image run: | - docker pull risingwavelabs/risingwave:v2.5.0 - docker pull postgres:16-alpine + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull risingwavelabs/risingwave:v2.5.0 + pull postgres:16-alpine - name: Run go mod tidy run: go mod tidy @@ -542,11 +569,15 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - name: Pre-pull Python image - run: docker pull python:3 + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull python:3 - name: Run go mod tidy run: go mod tidy @@ -608,14 +639,17 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - - name: Pre-pull Python image - run: docker pull python:3 - - - name: Pre-pull LocalStack image (if needed) - run: docker pull localstack/localstack:latest || true + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull python:3 + # localstack is optional; best-effort pull + for i in 1 2 3; do docker pull localstack/localstack:latest && break; sleep 15; done - name: Run go mod tidy run: go mod tidy @@ -677,14 +711,16 @@ jobs: go-version-file: 'go.mod' id: go - - name: Set up Docker - uses: docker/setup-buildx-action@v4 + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - - name: Pre-pull Unity Catalog image - run: docker pull unitycatalog/unitycatalog:v0.4.0 - - - name: Pre-pull Python image for delta-rs writer - run: docker pull python:3.11-slim + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull unitycatalog/unitycatalog:v0.4.0 + pull python:3.11-slim - name: Run go mod tidy run: go mod tidy