style: align const formatting in sra_transport.go

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.github/pull_request_template.md

@@ -12,7 +12,7 @@
 # Checks
 - [ ] I have added unit tests if possible.
 - [ ] I will add related wiki document changes and link to this PR after merging.
-- [ ] All AI code review comments have been addressed. No more comments to fix if reviewed again. Reviewer may request additional gemini and copilot reviews.
+- [ ] All AI code review comments have been addressed. No more comments to fix if reviewed again.
 
 # Checks for AI generated PRs
 - [ ] I have reviewed every line of code.

.github/workflows/container_dev.yml

@@ -3,6 +3,7 @@ name: "docker: build dev containers"
 on:
   push:
     branches: [ master ]
+  workflow_dispatch: {}
 
 permissions:
   contents: read

.github/workflows/container_latest.yml

@@ -4,58 +4,22 @@ on:
   push:
     tags:
       - '*'
-  workflow_dispatch:
-    inputs:
-      source_ref:
-        description: 'Git ref to build (branch, tag, or commit SHA)'
-        required: true
-        default: 'master'
-      image_tag:
-        description: 'Docker tag to publish (without variant suffix)'
-        required: true
-        default: 'latest'
-      variant:
-        description: 'Variant to build manually'
-        required: true
-        type: choice
-        default: all
-        options:
-          - all
-          - standard
-          - large_disk
+  workflow_dispatch: {}
 
 permissions:
   contents: read
 
 jobs:
-  setup:
-    runs-on: ubuntu-latest
-    outputs:
-      variants: ${{ steps.set-variants.outputs.variants }}
-    steps:
-      - name: Select variants for this run
-        id: set-variants
-        run: |
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.variant }}" != "all" ]; then
-            variants="[\"${{ github.event.inputs.variant }}\"]"
-          else
-            variants='["standard","large_disk"]'
-          fi
-          echo "variants=$variants" >> "$GITHUB_OUTPUT"
-
   build:
-    needs: [setup]
     runs-on: ubuntu-latest
     strategy:
       matrix:
         platform: [amd64, arm64, arm, 386]
-        variant: ${{ fromJSON(needs.setup.outputs.variants) }}
+        variant: [standard, large_disk]
 
     steps:
       - name: Checkout
         uses: actions/checkout@v6
-        with:
-          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
       - name: Free Disk Space
         run: |
           echo "Available disk space before cleanup:"
@@ -91,7 +55,7 @@ jobs:
           images: |
             chrislusf/seaweedfs
             ghcr.io/chrislusf/seaweedfs
-          tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
+          tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
           labels: |
             org.opencontainers.image.title=seaweedfs
             org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
@@ -133,13 +97,13 @@ jobs:
           file: ./docker/Dockerfile.go_build
           platforms: linux/${{ matrix.platform }}
           # Push to GHCR only during build to avoid Docker Hub rate limits
-          tags: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
+          tags: ghcr.io/chrislusf/seaweedfs:latest${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
           labels: ${{ steps.docker_meta.outputs.labels }}
           cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.platform }}
           build-args: |
             BUILDKIT_INLINE_CACHE=1
-            BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
+            BRANCH=${{ github.sha }}
             ${{ steps.config.outputs.build_args }}
       - name: Clean up build artifacts
         if: always()
@@ -151,16 +115,14 @@ jobs:
 
   create-manifest:
     runs-on: ubuntu-latest
-    needs: [setup, build]
+    needs: [build]
     if: github.event_name != 'pull_request'
     strategy:
       matrix:
-        variant: ${{ fromJSON(needs.setup.outputs.variants) }}
+        variant: [standard, large_disk]
     steps:
       - name: Checkout
         uses: actions/checkout@v6
-        with:
-          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
       
       - name: Configure variant
         id: config
@@ -178,7 +140,7 @@ jobs:
           images: |
             chrislusf/seaweedfs
             ghcr.io/chrislusf/seaweedfs
-          tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
+          tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -200,15 +162,14 @@ jobs:
       - name: Create and push manifest
         run: |
           SUFFIX="${{ steps.config.outputs.tag_suffix }}"
-          BASE_TAG="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}"
           
           # Create manifest on GHCR first (no rate limits)
           echo "Creating GHCR manifest (no rate limits)..."
-          docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
+          docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} \
+            ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
+            ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
+            ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
+            ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
           
           # Copy the complete multi-arch image from GHCR to Docker Hub
           # This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
@@ -244,16 +205,16 @@ jobs:
           # Use crane or skopeo to copy, fallback to docker if not available
           if command -v crane &> /dev/null; then
             echo "Using crane to copy..."
-            retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
+            retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} chrislusf/seaweedfs:latest${SUFFIX}
           elif command -v skopeo &> /dev/null; then
             echo "Using skopeo to copy..."
-            retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
+            retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} docker://chrislusf/seaweedfs:latest${SUFFIX}
           else
             echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
             # Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
-            retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
-              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
+            retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:latest${SUFFIX} \
+              ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
+              ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
+              ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
+              ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
           fi

.github/workflows/container_release1.yml

@@ -0,0 +1,152 @@
+name: "docker: build release containers for normal volume"
+
+# DISABLED: Merged into container_release_unified.yml
+on:
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [amd64, arm64, arm, 386]
+        include:
+          - platform: amd64
+            qemu: false
+          - platform: arm64
+            qemu: true
+          - platform: arm
+            qemu: true
+          - platform: 386
+            qemu: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Free Disk Space
+        run: |
+          echo "Available disk space before cleanup:"
+          df -h
+          # Remove pre-installed tools
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          # Clean package managers
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+          # Clean Docker aggressively
+          sudo docker system prune -af --volumes
+          # Clean Go cache if it exists
+          [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
+          [ -d /go/pkg ] && rm -rf /go/pkg || true
+          echo "Available disk space after cleanup:"
+          df -h
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: chrislusf/seaweedfs
+          tags: type=ref,event=tag
+          flavor: latest=false
+      - name: Set up QEMU
+        if: matrix.qemu
+        uses: docker/setup-qemu-action@v3
+      - name: Create BuildKit config
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config: /tmp/buildkitd.toml
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build ${{ matrix.platform }}
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_BUILDKIT: 1
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/${{ matrix.platform }}
+          tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+            BRANCH=${{ github.sha }}
+      - name: Clean up build artifacts
+        if: always()
+        run: |
+          # Clean up Docker build cache and temporary files
+          sudo docker system prune -f
+          # Remove Go build cache
+          sudo rm -rf /tmp/go-build*
+
+  create-manifest:
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: github.event_name != 'pull_request'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: chrislusf/seaweedfs
+          tags: type=ref,event=tag
+          flavor: latest=false
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Create and push manifest
+        run: |
+          # Function to retry command with exponential backoff
+          retry_with_backoff() {
+            local max_attempts=5
+            local timeout=1
+            local attempt=1
+            local exit_code=0
+            
+            while [ $attempt -le $max_attempts ]; do
+              if "$@"; then
+                return 0
+              else
+                exit_code=$?
+              fi
+              
+              if [ $attempt -lt $max_attempts ]; then
+                echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
+                sleep $timeout
+                timeout=$((timeout * 2))
+              fi
+              
+              attempt=$((attempt + 1))
+            done
+            
+            echo "Command failed after $max_attempts attempts" >&2
+            return $exit_code
+          }
+          
+          # Create manifest with retry
+          retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
+            ${{ steps.docker_meta.outputs.tags }}-amd64 \
+            ${{ steps.docker_meta.outputs.tags }}-arm64 \
+            ${{ steps.docker_meta.outputs.tags }}-arm \
+            ${{ steps.docker_meta.outputs.tags }}-386

.github/workflows/container_release2.yml

@@ -0,0 +1,153 @@
+name: "docker: build release containers for large volume"
+
+# DISABLED: Merged into container_release_unified.yml
+on:
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [amd64, arm64, arm, 386]
+        include:
+          - platform: amd64
+            qemu: false
+          - platform: arm64
+            qemu: true
+          - platform: arm
+            qemu: true
+          - platform: 386
+            qemu: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Free Disk Space
+        run: |
+          echo "Available disk space before cleanup:"
+          df -h
+          # Remove pre-installed tools
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          # Clean package managers
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+          # Clean Docker aggressively
+          sudo docker system prune -af --volumes
+          # Clean Go cache if it exists
+          [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
+          [ -d /go/pkg ] && rm -rf /go/pkg || true
+          echo "Available disk space after cleanup:"
+          df -h
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: chrislusf/seaweedfs
+          tags: type=ref,event=tag,suffix=_large_disk
+          flavor: latest=false
+      - name: Set up QEMU
+        if: matrix.qemu
+        uses: docker/setup-qemu-action@v3
+      - name: Create BuildKit config
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config: /tmp/buildkitd.toml
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build ${{ matrix.platform }}
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_BUILDKIT: 1
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build
+          build-args: |
+            TAGS=5BytesOffset
+            BUILDKIT_INLINE_CACHE=1
+            BRANCH=${{ github.sha }}
+          platforms: linux/${{ matrix.platform }}
+          tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Clean up build artifacts
+        if: always()
+        run: |
+          # Clean up Docker build cache and temporary files
+          sudo docker system prune -f
+          # Remove Go build cache
+          sudo rm -rf /tmp/go-build*
+
+  create-manifest:
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: github.event_name != 'pull_request'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: chrislusf/seaweedfs
+          tags: type=ref,event=tag,suffix=_large_disk
+          flavor: latest=false
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Create and push manifest
+        run: |
+          # Function to retry command with exponential backoff
+          retry_with_backoff() {
+            local max_attempts=5
+            local timeout=1
+            local attempt=1
+            local exit_code=0
+            
+            while [ $attempt -le $max_attempts ]; do
+              if "$@"; then
+                return 0
+              else
+                exit_code=$?
+              fi
+              
+              if [ $attempt -lt $max_attempts ]; then
+                echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
+                sleep $timeout
+                timeout=$((timeout * 2))
+              fi
+              
+              attempt=$((attempt + 1))
+            done
+            
+            echo "Command failed after $max_attempts attempts" >&2
+            return $exit_code
+          }
+          
+          # Create manifest with retry
+          retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
+            ${{ steps.docker_meta.outputs.tags }}-amd64 \
+            ${{ steps.docker_meta.outputs.tags }}-arm64 \
+            ${{ steps.docker_meta.outputs.tags }}-arm \
+            ${{ steps.docker_meta.outputs.tags }}-386

.github/workflows/container_release3.yml

@@ -0,0 +1,73 @@
+name: "docker: build release containers for rocksdb"
+
+# DISABLED: Merged into container_release_unified.yml
+on:
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
+
+permissions:
+  contents: read
+
+jobs:
+
+  build-large-release-container_rocksdb:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v6
+      -
+        name: Free Disk Space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+          sudo docker system prune -af
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: chrislusf/seaweedfs
+          tags: type=ref,event=tag,suffix=_large_disk_rocksdb
+          flavor: latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Create BuildKit config
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config: /tmp/buildkitd.toml
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.rocksdb_large
+          build-args: |
+            BRANCH=${{ github.sha }}
+          platforms: linux/amd64
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/container_release4.yml

@@ -0,0 +1,72 @@
+name: "docker: build release containers for all tags"
+
+# DISABLED: Merged into container_release_unified.yml
+on:
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
+
+permissions:
+  contents: read
+
+jobs:
+  build-default-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v6
+      -
+        name: Free Disk Space
+        run: |
+          echo "Before cleanup:"
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker system prune -af
+          echo "After cleanup:"
+          df -h
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag,suffix=_full
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build
+          build-args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
+          platforms: linux/amd64
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/container_release5.yml

@@ -0,0 +1,62 @@
+name: "docker: build release containers for all tags and large volume"
+
+# DISABLED: Merged into container_release_unified.yml
+on:
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
+
+permissions:
+  contents: read
+
+jobs:
+  build-default-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v6
+      -
+        name: Free Disk Space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+          sudo docker system prune -af
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: chrislusf/seaweedfs
+          tags: type=ref,event=tag,suffix=_large_disk_full
+          flavor: latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build
+          build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
+          platforms: linux/amd64
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/container_release_unified.yml

@@ -4,35 +4,11 @@ on:
   push:
     tags:
       - '*'
-  workflow_dispatch:
-    inputs:
-      variant:
-        description: 'Variant to build manually'
-        required: true
-        type: choice
-        default: all
-        options:
-          - all
-          - normal
-          - large_disk
-          - full
-          - large_disk_full
-          - rocksdb
-      release_tag:
-        description: 'Release tag to publish (e.g. 3.93)'
-        required: true
-        default: ''
-      rocksdb_version:
-        description: 'RocksDB git tag to use when variant=rocksdb'
-        required: false
-        default: 'v10.10.1'
+  workflow_dispatch: {}
 
 permissions:
   contents: read
 
-env:
-  RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
-
 # Limit concurrent builds to avoid rate limits
 concurrency:
   group: release-${{ github.ref }}
@@ -83,11 +59,9 @@ jobs:
             
     steps:
       - name: Checkout
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: actions/checkout@v6
         
       - name: Free Disk Space
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |
           echo "Available disk space before cleanup:"
           df -h
@@ -101,14 +75,13 @@ jobs:
           df -h
           
       - name: Docker meta
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         id: docker_meta
         uses: docker/metadata-action@v5
         with:
           images: |
             chrislusf/seaweedfs
             ghcr.io/chrislusf/seaweedfs
-          tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
+          tags: type=ref,event=tag,suffix=${{ matrix.tag_suffix }}
           flavor: latest=false
           labels: |
             org.opencontainers.image.title=seaweedfs
@@ -116,11 +89,10 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
             
       - name: Set up QEMU
-        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
+        if: contains(matrix.platforms, 'arm')
         uses: docker/setup-qemu-action@v3
         
       - name: Create BuildKit config
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |
           cat > /tmp/buildkitd.toml <<EOF
           [registry."docker.io"]
@@ -128,20 +100,19 @@ jobs:
           EOF
           
       - name: Set up Docker Buildx
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: docker/setup-buildx-action@v3
         with:
           buildkitd-config: /tmp/buildkitd.toml
           
       - name: Login to Docker Hub
-        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
-        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -149,7 +120,6 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
           
       - name: Build and push ${{ matrix.variant }}
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: docker/build-push-action@v6
         env:
           DOCKER_BUILDKIT: 1
@@ -160,7 +130,7 @@ jobs:
           platforms: ${{ matrix.platforms }}
           # Push to GHCR to avoid Docker Hub rate limits on pulls
           tags: |
-            ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
+            ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
           labels: ${{ steps.docker_meta.outputs.labels }}
           cache-from: type=gha,scope=${{ matrix.variant }}
           cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
@@ -168,10 +138,9 @@ jobs:
             ${{ matrix.build_args }}
             BUILDKIT_INLINE_CACHE=1
             BRANCH=${{ github.sha }}
-            ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
             
       - name: Clean up build artifacts
-        if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
+        if: always()
         run: |
           sudo docker system prune -f
           sudo rm -rf /tmp/go-build*
@@ -197,14 +166,12 @@ jobs:
             
     steps:
       - name: Login to Docker Hub
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -212,7 +179,6 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
           
       - name: Install crane
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |
           cd $(mktemp -d)
           curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
@@ -220,7 +186,6 @@ jobs:
           crane version
           
       - name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
-        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |
           # Function to retry with exponential backoff
           retry_with_backoff() {
@@ -253,15 +218,14 @@ jobs:
           # This is much more efficient than pulling/pushing individual arch images
           echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
           retry_with_backoff crane copy \
-            ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
-            chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
+            ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }} \
+            chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
           
           echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
 
   helm-release:
     runs-on: ubuntu-latest
     needs: [copy-to-dockerhub]
-    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     permissions:
       contents: write
       pages: write
@@ -275,3 +239,7 @@ jobs:
           target_dir: helm
           branch: gh-pages
           helm_version: "3.18.4"
+
+
+
+

.github/workflows/container_rocksdb_version.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
     inputs:
       rocksdb_version:
-        description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
+        description: 'RocksDB git tag or branch to build (e.g. v10.5.1)'
         required: true
-        default: 'v10.10.1'
+        default: 'v10.5.1'
       seaweedfs_ref:
         description: 'SeaweedFS git tag, branch, or commit to build'
         required: true

.github/workflows/go.yml

@@ -15,20 +15,24 @@ permissions:
 
 jobs:
 
-  vet:
-    name: Go Vet
+  build:
+    name: Build
     runs-on: ubuntu-latest
     steps:
+
     - name: Set up Go 1.x
       uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
       with:
         go-version: ^1.13
       id: go
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
+
     - name: Get dependencies
       run: |
         cd weed; go get -v -t -d ./...
+
     - name: Go Vet (excluding protobuf lock copying)
       run: |
         cd weed
@@ -38,30 +42,8 @@ jobs:
         # Fail only if there are actual vet errors (not counting the filtered lock warnings)
         if grep -q "vet:" vet-output.txt; then exit 1; fi
 
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
-      with:
-        go-version: ^1.13
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
     - name: Build
       run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
 
-  test:
-    name: Test
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
-      with:
-        go-version: ^1.13
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
     - name: Test
       run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...

.github/workflows/helm_manual_release.yml

@@ -1,25 +0,0 @@
-name: "helm: manual release"
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  pages: write
-
-jobs:
-  helm-release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Publish Helm charts
-        uses: stefanprodan/helm-gh-pages@v1.7.0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          charts_dir: k8s/charts
-          target_dir: helm
-          branch: gh-pages
-          helm_version: "3.18.4"

.github/workflows/java_integration_tests.yml

@@ -111,7 +111,7 @@ jobs:
           
           # Wait for S3 API
           for i in {1..30}; do
-            if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
+            if curl -s http://localhost:8333/ > /dev/null 2>&1; then
               echo "✓ S3 API is ready"
               break
             fi

.github/workflows/s3-go-tests.yml

@@ -461,8 +461,6 @@ jobs:
           export S3_ENDPOINT="http://localhost:8006"
           export S3_ACCESS_KEY="0555b35654ad1656d804"
           export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
-          export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
-          export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
           
           # Run the specific test that is equivalent to AWS S3 tagging behavior
           make test-with-server || {

.github/workflows/s3-keycloak-tests.yml

@@ -97,7 +97,7 @@ jobs:
           # Verify service accessibility
           echo "=== Verifying Service Accessibility ==="
           curl -f http://localhost:8080/realms/master
-          curl -s http://localhost:8333/healthz
+          curl -s http://localhost:8333
           echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
           
           # Run Keycloak-specific tests

.github/workflows/s3-spark-tests.yml

@@ -1,80 +0,0 @@
-name: "S3 Spark Integration Tests"
-
-on:
-  pull_request:
-    paths:
-      - 'weed/s3api/**'
-      - 'test/s3/spark/**'
-      - 'test/s3tables/testutil/**'
-      - '.github/workflows/s3-spark-tests.yml'
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.head_ref }}/s3-spark-tests
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  s3-spark-issue-repro-tests:
-    name: S3 Spark Issue Reproduction Tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 45
-
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v6
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: 'go.mod'
-        id: go
-
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      - name: Install SeaweedFS
-        run: |
-          go install -buildvcs=false ./weed
-
-      - name: Pre-pull Spark image
-        run: docker pull apache/spark:3.5.8
-
-      - name: Run S3 Spark integration tests
-        working-directory: test/s3/spark
-        timeout-minutes: 35
-        run: |
-          set -x
-          set -o pipefail
-          echo "=== System Information ==="
-          uname -a
-          free -h
-          df -h
-          echo "=== Starting S3 Spark Integration Tests ==="
-
-          go test -v -timeout 30m . 2>&1 | tee test-output.log || {
-            echo "S3 Spark integration tests failed"
-            exit 1
-          }
-
-      - name: Show test output on failure
-        if: failure()
-        working-directory: test/s3/spark
-        run: |
-          echo "=== Test Output ==="
-          if [ -f test-output.log ]; then
-            tail -200 test-output.log
-          fi
-
-          echo "=== Process information ==="
-          ps aux | grep -E "(weed|test|docker|spark)" || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v6
-        with:
-          name: s3-spark-test-logs
-          path: test/s3/spark/test-output.log
-          retention-days: 3

.github/workflows/s3-tables-tests.yml

@@ -27,9 +27,6 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Run go mod tidy
-        run: go mod tidy
-
       - name: Install SeaweedFS
         run: |
           go install -buildvcs=false ./weed
@@ -87,9 +84,6 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Run go mod tidy
-        run: go mod tidy
-
       - name: Run Iceberg Catalog Integration Tests
         timeout-minutes: 25
         working-directory: test/s3tables/catalog
@@ -149,9 +143,6 @@ jobs:
       - name: Pre-pull Trino image
         run: docker pull trinodb/trino:479
 
-      - name: Run go mod tidy
-        run: go mod tidy
-
       - name: Install SeaweedFS
         run: |
           go install -buildvcs=false ./weed
@@ -194,275 +185,6 @@ jobs:
           path: test/s3tables/catalog_trino/test-output.log
           retention-days: 3
 
-  spark-iceberg-catalog-tests:
-    name: Spark Iceberg Catalog Integration Tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v6
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: 'go.mod'
-        id: go
-
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      - name: Pre-pull Spark image
-        run: docker pull apache/spark:3.5.1
-
-      - name: Run go mod tidy
-        run: go mod tidy
-
-      - name: Install SeaweedFS
-        run: |
-          go install -buildvcs=false ./weed
-
-      - name: Run Spark Iceberg Catalog Integration Tests
-        timeout-minutes: 25
-        working-directory: test/s3tables/catalog_spark
-        run: |
-          set -x
-          set -o pipefail
-          echo "=== System Information ==="
-          uname -a
-          free -h
-          df -h
-          echo "=== Starting Spark Iceberg Catalog Tests ==="
-          
-          # Run Spark + Iceberg catalog integration tests
-          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
-            echo "Spark Iceberg catalog integration tests failed"
-            exit 1
-          }
-
-      - name: Show test output on failure
-        if: failure()
-        working-directory: test/s3tables/catalog_spark
-        run: |
-          echo "=== Test Output ==="
-          if [ -f test-output.log ]; then
-            tail -200 test-output.log
-          fi
-          
-          echo "=== Process information ==="
-          ps aux | grep -E "(weed|test|docker)" || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v6
-        with:
-          name: spark-iceberg-catalog-test-logs
-          path: test/s3tables/catalog_spark/test-output.log
-          retention-days: 3
-
-  risingwave-catalog-tests:
-    name: RisingWave Catalog Integration Tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v6
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: 'go.mod'
-        id: go
-
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      - name: Pre-pull RisingWave image
-        run: |
-          docker pull risingwavelabs/risingwave:v2.5.0
-          docker pull postgres:16-alpine
-
-      - name: Run go mod tidy
-        run: go mod tidy
-
-      - name: Install SeaweedFS
-        run: |
-          go install -buildvcs=false ./weed
-
-      - name: Run RisingWave Catalog Integration Tests
-        timeout-minutes: 25
-        working-directory: test/s3tables/catalog_risingwave
-        run: |
-          set -x
-          set -o pipefail
-          echo "=== System Information ==="
-          uname -a
-          free -h
-          df -h
-          echo "=== Starting RisingWave Catalog Tests ==="
-          
-          # Run RisingWave catalog integration tests
-          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
-            echo "RisingWave catalog integration tests failed"
-            exit 1
-          }
-
-      - name: Show test output on failure
-        if: failure()
-        working-directory: test/s3tables/catalog_risingwave
-        run: |
-          echo "=== Test Output ==="
-          if [ -f test-output.log ]; then
-            tail -200 test-output.log
-          fi
-          
-          echo "=== Process information ==="
-          ps aux | grep -E "(weed|test|docker)" || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v6
-        with:
-          name: risingwave-catalog-test-logs
-          path: test/s3tables/catalog_risingwave/test-output.log
-          retention-days: 3
-
-  sts-integration-tests:
-    name: STS Integration Tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v6
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: 'go.mod'
-        id: go
-
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      - name: Pre-pull Python image
-        run: docker pull python:3
-
-      - name: Run go mod tidy
-        run: go mod tidy
-
-      - name: Install SeaweedFS
-        run: |
-          go install -buildvcs=false ./weed
-
-      - name: Run STS Integration Tests
-        timeout-minutes: 25
-        working-directory: test/s3tables/sts_integration
-        run: |
-          set -x
-          set -o pipefail
-          echo "=== System Information ==="
-          uname -a
-          free -h
-          df -h
-          echo "=== Starting STS Integration Tests ==="
-          
-          # Run STS integration tests
-          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
-            echo "STS integration tests failed"
-            exit 1
-          }
-
-      - name: Show test output on failure
-        if: failure()
-        working-directory: test/s3tables/sts_integration
-        run: |
-          echo "=== Test Output ==="
-          if [ -f test-output.log ]; then
-            tail -200 test-output.log
-          fi
-          
-          echo "=== Process information ==="
-          ps aux | grep -E "(weed|test|docker)" || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v6
-        with:
-          name: sts-integration-test-logs
-          path: test/s3tables/sts_integration/test-output.log
-          retention-days: 3
-
-  lakekeeper-integration-tests:
-    name: Lakekeeper Integration Tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v6
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: 'go.mod'
-        id: go
-
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      - name: Pre-pull Python image
-        run: docker pull python:3
-        
-      - name: Pre-pull LocalStack image (if needed)
-        run: docker pull localstack/localstack:latest || true
-
-      - name: Run go mod tidy
-        run: go mod tidy
-
-      - name: Install SeaweedFS
-        run: |
-          go install -buildvcs=false ./weed
-
-      - name: Run Lakekeeper Integration Tests
-        timeout-minutes: 25
-        working-directory: test/s3tables/lakekeeper
-        run: |
-          set -x
-          set -o pipefail
-          echo "=== System Information ==="
-          uname -a
-          free -h
-          df -h
-          echo "=== Starting Lakekeeper Integration Tests ==="
-          
-          # Run Lakekeeper integration tests
-          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
-            echo "Lakekeeper integration tests failed"
-            exit 1
-          }
-
-      - name: Show test output on failure
-        if: failure()
-        working-directory: test/s3tables/lakekeeper
-        run: |
-          echo "=== Test Output ==="
-          if [ -f test-output.log ]; then
-            tail -200 test-output.log
-          fi
-          
-          echo "=== Process information ==="
-          ps aux | grep -E "(weed|test|docker)" || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v6
-        with:
-          name: lakekeeper-integration-test-logs
-          path: test/s3tables/lakekeeper/test-output.log
-          retention-days: 3
-
   s3-tables-build-verification:
     name: S3 Tables Build Verification
     runs-on: ubuntu-22.04

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -35,7 +35,7 @@ jobs:
           set -e
           mkdir -p /tmp/data
           ./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
-          until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
+          until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
 
       - name: Setup Caddy
         run: |
@@ -54,7 +54,7 @@ jobs:
       - name: Start Caddy
         run: |
           ./caddy start
-          until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
+          until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
 
       - name: Create Bucket
         run: |

.github/workflows/volume-server-integration-tests.yml

@@ -1,122 +0,0 @@
-name: "Volume Server Integration Tests"
-
-on:
-  pull_request:
-    branches: [ master ]
-    paths:
-      - 'test/volume_server/**'
-      - 'weed/server/**'
-      - 'weed/storage/**'
-      - 'weed/pb/volume_server.proto'
-      - 'weed/pb/volume_server_pb/**'
-      - '.github/workflows/volume-server-integration-tests.yml'
-  push:
-    branches: [ master, main ]
-    paths:
-      - 'test/volume_server/**'
-      - 'weed/server/**'
-      - 'weed/storage/**'
-      - 'weed/pb/volume_server.proto'
-      - 'weed/pb/volume_server_pb/**'
-      - '.github/workflows/volume-server-integration-tests.yml'
-
-concurrency:
-  group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-env:
-  GO_VERSION: '1.24'
-  TEST_TIMEOUT: '30m'
-
-jobs:
-  volume-server-integration-tests:
-    name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
-    runs-on: ubuntu-22.04
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        test-type: [grpc, http]
-        shard: [1, 2, 3]
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v6
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - name: Build SeaweedFS binary
-        run: |
-          cd weed
-          go build -o weed .
-          chmod +x weed
-          ./weed version
-
-      - name: Run volume server integration tests
-        env:
-          WEED_BINARY: ${{ github.workspace }}/weed/weed
-        run: |
-          if [ "${{ matrix.test-type }}" == "grpc" ]; then
-            if [ "${{ matrix.shard }}" == "1" ]; then
-              TEST_PATTERN="^Test[A-H]"
-            elif [ "${{ matrix.shard }}" == "2" ]; then
-              TEST_PATTERN="^Test[I-S]"
-            else
-              TEST_PATTERN="^Test[T-Z]"
-            fi
-          else
-            if [ "${{ matrix.shard }}" == "1" ]; then
-              TEST_PATTERN="^Test[A-G]"
-            elif [ "${{ matrix.shard }}" == "2" ]; then
-              TEST_PATTERN="^Test[H-R]"
-            else
-              TEST_PATTERN="^Test[S-Z]"
-            fi
-          fi
-          echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
-          go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
-
-      - name: Collect logs on failure
-        if: failure()
-        run: |
-          mkdir -p /tmp/volume-server-it-logs
-          find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true
-
-      - name: Archive logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v6
-        with:
-          name: volume-server-integration-test-logs
-          path: /tmp/volume-server-it-logs/
-          if-no-files-found: warn
-          retention-days: 7
-
-      - name: Test summary
-        if: always()
-        run: |
-          if [ "${{ matrix.test-type }}" == "grpc" ]; then
-            if [ "${{ matrix.shard }}" == "1" ]; then
-              TEST_PATTERN="^Test[A-H]"
-            elif [ "${{ matrix.shard }}" == "2" ]; then
-              TEST_PATTERN="^Test[I-S]"
-            else
-              TEST_PATTERN="^Test[T-Z]"
-            fi
-          else
-            if [ "${{ matrix.shard }}" == "1" ]; then
-              TEST_PATTERN="^Test[A-G]"
-            elif [ "${{ matrix.shard }}" == "2" ]; then
-              TEST_PATTERN="^Test[H-R]"
-            else
-              TEST_PATTERN="^Test[S-Z]"
-            fi
-          fi
-          echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
-          echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
-          echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"

Makefile

@@ -40,7 +40,7 @@ test: admin-generate
 # Admin component targets
 admin-generate:
 	@echo "Generating admin component templates..."
-	@templ generate
+	@cd $(ADMIN_DIR) && $(MAKE) generate
 
 admin-build: admin-generate
 	@echo "Building admin component..."

docker/Dockerfile.rocksdb_dev_env

@@ -3,7 +3,7 @@ FROM golang:1.24 AS builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ARG ROCKSDB_VERSION=v10.10.1
+ARG ROCKSDB_VERSION=v10.5.1
 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
 
 # build RocksDB

docker/Dockerfile.rocksdb_large

@@ -3,7 +3,7 @@ FROM golang:1.24 AS builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ARG ROCKSDB_VERSION=v10.10.1
+ARG ROCKSDB_VERSION=v10.5.1
 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
 
 # build RocksDB

go.mod

@@ -46,10 +46,10 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.18.4
+	github.com/klauspost/compress v1.18.3
 	github.com/klauspost/reedsolomon v1.13.0
 	github.com/kurin/blazer v0.5.3
-	github.com/linxGnu/grocksdb v1.10.7
+	github.com/linxGnu/grocksdb v1.10.3
 	github.com/mailru/easyjson v0.9.1 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -86,20 +86,20 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.7
+	go.etcd.io/etcd/client/v3 v3.6.6
 	go.mongodb.org/mongo-driver v1.17.6
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.44.0
 	gocloud.dev/pubsub/natspubsub v0.44.0
 	gocloud.dev/pubsub/rabbitpubsub v0.44.0
-	golang.org/x/crypto v0.48.0
+	golang.org/x/crypto v0.47.0
 	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
 	golang.org/x/image v0.35.0
 	golang.org/x/net v0.49.0
 	golang.org/x/oauth2 v0.34.0
-	golang.org/x/sys v0.41.0
-	golang.org/x/text v0.34.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/sys v0.40.0
+	golang.org/x/text v0.33.0 // indirect
+	golang.org/x/tools v0.40.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.258.0
 	google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
@@ -130,7 +130,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.42.0
+	github.com/getsentry/sentry-go v0.40.0
 	github.com/gin-contrib/sessions v1.0.4
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-ldap/ldap/v3 v3.4.12
@@ -139,10 +139,10 @@ require (
 	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
 	github.com/hashicorp/vault/api v1.22.0
-	github.com/jhump/protoreflect v1.18.0
+	github.com/jhump/protoreflect v1.17.0
 	github.com/lib/pq v1.11.1
 	github.com/linkedin/goavro/v2 v2.14.1
-	github.com/mattn/go-sqlite3 v1.14.34
+	github.com/mattn/go-sqlite3 v1.14.33
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.26.4
@@ -150,12 +150,11 @@ require (
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rclone/rclone v1.72.1
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.18.0
+	github.com/redis/go-redis/v9 v9.17.2
 	github.com/schollz/progressbar/v3 v3.19.0
 	github.com/seaweedfs/go-fuse/v2 v2.9.1
 	github.com/shirou/gopsutil/v4 v4.26.1
 	github.com/tarantool/go-tarantool/v2 v2.4.1
-	github.com/testcontainers/testcontainers-go v0.39.0
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
@@ -175,9 +174,7 @@ require (
 	atomicgo.dev/schedule v0.1.0 // indirect
 	cloud.google.com/go/longrunning v0.7.0 // indirect
 	cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
-	dario.cat/mergo v1.0.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect
 	github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
@@ -191,6 +188,7 @@ require (
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
 	github.com/boombuler/barcode v1.1.0 // indirect
+	github.com/bufbuild/protocompile v0.14.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/clipperhouse/stringish v0.1.1 // indirect
@@ -201,17 +199,8 @@ require (
 	github.com/cockroachdb/redact v1.1.5 // indirect
 	github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect
 	github.com/containerd/console v1.0.5 // indirect
-	github.com/containerd/errdefs v1.0.0 // indirect
-	github.com/containerd/errdefs/pkg v0.3.0 // indirect
-	github.com/containerd/log v0.1.0 // indirect
-	github.com/containerd/platforms v1.0.0-rc.1 // indirect
-	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/dave/dst v0.27.2 // indirect
 	github.com/diskfs/go-diskfs v1.7.0 // indirect
-	github.com/distribution/reference v0.6.0 // indirect
-	github.com/docker/docker v28.5.0+incompatible // indirect
-	github.com/docker/go-connections v0.6.0 // indirect
-	github.com/docker/go-units v0.5.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
 	github.com/goccy/go-yaml v1.18.0 // indirect
@@ -230,30 +219,18 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jaegertracing/jaeger v1.47.0 // indirect
-	github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
-	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
-	github.com/moby/docker-image-spec v1.3.1 // indirect
-	github.com/moby/go-archive v0.1.0 // indirect
-	github.com/moby/patternmatcher v0.6.0 // indirect
-	github.com/moby/sys/sequential v0.6.0 // indirect
-	github.com/moby/sys/user v0.4.0 // indirect
-	github.com/moby/sys/userns v0.1.0 // indirect
-	github.com/moby/term v0.5.2 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/openzipkin/zipkin-go v0.4.3 // indirect
 	github.com/parquet-go/bitpack v1.0.0 // indirect
 	github.com/parquet-go/jsonlite v1.0.0 // indirect
-	github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pierrre/geohash v1.0.0 // indirect
 	github.com/pquerna/otp v1.5.0 // indirect
 	github.com/pterm/pterm v0.12.81 // indirect
@@ -281,8 +258,8 @@ require (
 	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.32.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect
+	golang.org/x/mod v0.31.0 // indirect
+	golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc // indirect
 	gonum.org/v1/gonum v0.16.0 // indirect
 )
 
@@ -297,7 +274,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
 	github.com/Azure/go-ntlmssp v0.1.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
@@ -492,7 +469,7 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.7 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.6 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
@@ -505,7 +482,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	golang.org/x/arch v0.20.0 // indirect
-	golang.org/x/term v0.40.0 // indirect
+	golang.org/x/term v0.39.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect

go.sum

@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -904,8 +904,6 @@ github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X
 github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
 github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
-github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
 github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
 github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=
@@ -1053,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
 github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
-github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
+github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
+github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
 github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -1432,10 +1430,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/jhump/protoreflect v1.18.0 h1:TOz0MSR/0JOZ5kECB/0ufGnC2jdsgZ123Rd/k4Z5/2w=
-github.com/jhump/protoreflect v1.18.0/go.mod h1:ezWcltJIVF4zYdIFM+D/sHV4Oh5LNU08ORzCGfwvTz8=
-github.com/jhump/protoreflect/v2 v2.0.0-beta.1 h1:Dw1rslK/VotaUGYsv53XVWITr+5RCPXfvvlGrM/+B6w=
-github.com/jhump/protoreflect/v2 v2.0.0-beta.1/go.mod h1:D9LBEowZyv8/iSu97FU2zmXG3JxVTmNw21mu63niFzU=
+github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94=
+github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
 github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -1487,8 +1483,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
+github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -1530,8 +1526,8 @@ github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
 github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
 github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
 github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
-github.com/linxGnu/grocksdb v1.10.7 h1:fCi4qvZWo04VgFwGWmO8HQJgUVounJBy+C2TMVPU/ho=
-github.com/linxGnu/grocksdb v1.10.7/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
+github.com/linxGnu/grocksdb v1.10.3 h1:0laII9AQ6kFxo5SjhdTfSh9EgF20piD6TMHK6YuDm+4=
+github.com/linxGnu/grocksdb v1.10.3/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
 github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4=
 github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
@@ -1564,8 +1560,8 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
 github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
-github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
+github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -1704,9 +1700,8 @@ github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uC
 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=
 github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw=
 github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
-github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 h1:KPpdlQLZcHfTMQRi6bFQ7ogNO0ltFT4PmtwTLW4W+14=
-github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
 github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
 github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
 github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
@@ -1814,8 +1809,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
-github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
+github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
+github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
 github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
 github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
 github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
@@ -2121,12 +2116,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
 go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
-go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
-go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
+go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ=
+go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4=
 go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
 go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
-go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
-go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
+go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec=
+go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8=
 go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
 go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2228,8 +2223,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2295,8 +2290,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
-golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
+golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
+golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2525,11 +2520,11 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
-golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo=
-golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
+golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc h1:bH6xUXay0AIFMElXG2rQ4uiE+7ncwtiOdPfYK1NK2XA=
+golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2545,8 +2540,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2567,8 +2562,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2647,8 +2642,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
+golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
 golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
 golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2971,8 +2966,6 @@ gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
-gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.13"
+appVersion: "4.09"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.413
+version: 4.0.409

k8s/charts/seaweedfs/templates/admin/admin-service.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "seaweedfs.componentName" (list . "admin") }}
+  name: {{ printf "%s-admin" (include "seaweedfs.fullname" .) | trunc 63 | trimSuffix "-" }}
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml

@@ -134,7 +134,7 @@ spec:
               {{- if .Values.worker.adminServer }}
               -admin={{ .Values.worker.adminServer }} \
               {{- else }}
-              -admin={{ template "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
+              -admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
               {{- end }}
               -capabilities={{ .Values.worker.capabilities }} \
               -maxConcurrent={{ .Values.worker.maxConcurrent }} \

test/java/spark/docker-compose.yml

@@ -19,6 +19,9 @@ services:
       start_period: 10s
 
   seaweedfs-volume:
+    build:
+      context: ../../../docker
+      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-volume
     ports:
@@ -40,6 +43,9 @@ services:
       start_period: 10s
 
   seaweedfs-filer:
+    build:
+      context: ../../../docker
+      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-filer
     ports:

test/java/spark/pom.xml

@@ -116,7 +116,7 @@
             <dependency>
                 <groupId>org.apache.avro</groupId>
                 <artifactId>avro</artifactId>
-                <version>1.11.5</version>
+                <version>1.11.4</version>
             </dependency>
 
             <!-- Apache ZooKeeper - Fix CVEs -->

test/kafka/kafka-client-loadtest/tools/pom.xml

@@ -41,7 +41,7 @@
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
-            <version>1.11.5</version>
+            <version>1.11.4</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

test/s3/cors/Makefile

@@ -12,7 +12,6 @@ FILER_PORT := 8888
 TEST_TIMEOUT := 10m
 TEST_PATTERN := TestCORS
 SERVER_DIR := test-mini-data
-S3_CONFIG := s3_test_config.json
 
 # Default target
 help:
@@ -81,15 +80,13 @@ start-server: check-deps
 	@echo "🔍 DEBUG: Creating volume directory..."
 	@mkdir -p $(SERVER_DIR)
 	@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
-	@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
-	@env AWS_ACCESS_KEY_ID=some_access_key1 \
-	 AWS_SECRET_ACCESS_KEY=some_secret_key1 \
-	 $(WEED_BINARY) mini \
+	@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
+	@$(WEED_BINARY) mini \
 		-dir=$(SERVER_DIR) \
 		-s3.port=$(S3_PORT) \
 		-s3.config=$(S3_CONFIG) \
 		> weed-test.log 2>&1 & \
-		echo $$! > weed-server.pid
+		echo $$! > weed-test.pid
 	
 	@echo "Waiting for S3 server to be ready..."
 	@for i in $$(seq 1 30); do \
@@ -100,7 +97,7 @@ start-server: check-deps
 		sleep 1; \
 	done; \
 	echo "S3 server failed to start"; \
-	exit 1
+	exit 1 > weed-server.pid
 	@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
 	@echo "🔍 DEBUG: Checking if PID is still running..."
 	@sleep 2

test/s3/cors/s3_test_config.json

@@ -1,27 +0,0 @@
-{
-    "identities": [
-        {
-            "name": "anonymous",
-            "actions": [
-                "Read",
-                "List"
-            ]
-        },
-        {
-            "name": "admin",
-            "credentials": [
-                {
-                    "accessKey": "some_access_key1",
-                    "secretKey": "some_secret_key1"
-                }
-            ],
-            "actions": [
-                "Admin",
-                "Read",
-                "List",
-                "Tagging",
-                "Write"
-            ]
-        }
-    ]
-}

test/s3/iam/s3_sts_credential_prefix_test.go

@@ -1,82 +0,0 @@
-package iam
-
-import (
-	"encoding/xml"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestSTSTemporaryCredentialPrefix verifies that STS temporary credentials use ASIA prefix
-// This test ensures AWS compatibility - temporary credentials should use ASIA, not AKIA
-func TestSTSTemporaryCredentialPrefix(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	if !isSTSEndpointRunning(t) {
-		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
-	}
-
-	// Use test credentials from environment or fall back to defaults
-	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
-	if accessKey == "" {
-		accessKey = "admin"
-	}
-	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
-	if secretKey == "" {
-		secretKey = "admin"
-	}
-
-	t.Run("assume_role_returns_asia_prefix", func(t *testing.T) {
-		resp, err := callSTSAPIWithSigV4(t, url.Values{
-			"Action":          {"AssumeRole"},
-			"Version":         {"2011-06-15"},
-			"RoleArn":         {"arn:aws:iam::role/admin"},
-			"RoleSessionName": {"asia-prefix-test"},
-		}, accessKey, secretKey)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		body, err := io.ReadAll(resp.Body)
-		require.NoError(t, err)
-
-		if resp.StatusCode != http.StatusOK {
-			t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
-			t.Skip("AssumeRole not fully implemented yet")
-		}
-
-		var stsResp AssumeRoleTestResponse
-		err = xml.Unmarshal(body, &stsResp)
-		require.NoError(t, err, "Failed to parse response: %s", string(body))
-
-		creds := stsResp.Result.Credentials
-		require.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty")
-
-		// Verify ASIA prefix for temporary credentials
-		assert.True(t, strings.HasPrefix(creds.AccessKeyId, "ASIA"),
-			"Temporary credentials must use ASIA prefix (not AKIA for permanent keys), got: %s", creds.AccessKeyId)
-
-		// Verify it's NOT using AKIA (permanent credentials)
-		assert.False(t, strings.HasPrefix(creds.AccessKeyId, "AKIA"),
-			"Temporary credentials must NOT use AKIA prefix (that's for permanent IAM keys), got: %s", creds.AccessKeyId)
-
-		// Verify format: ASIA + 16 hex characters = 20 chars total
-		assert.Equal(t, 20, len(creds.AccessKeyId),
-			"Access key ID should be 20 characters (ASIA + 16 hex chars), got: %s", creds.AccessKeyId)
-
-		t.Logf("✓ Temporary credentials correctly use ASIA prefix: %s", creds.AccessKeyId)
-	})
-
-	t.Run("assume_role_with_web_identity_returns_asia_prefix", func(t *testing.T) {
-		// This test would require OIDC setup, so we'll skip it for now
-		// but the same ASIA prefix validation should apply
-		t.Skip("AssumeRoleWithWebIdentity requires OIDC provider setup")
-	})
-}

test/s3/normal/iam_test.go

@@ -1,6 +1,7 @@
 package example
 
 import (
+	"os"
 	"testing"
 	"time"
 
@@ -21,8 +22,10 @@ func TestIAMOperations(t *testing.T) {
 	// Set credentials before starting cluster
 	accessKey := "testkey123"
 	secretKey := "testsecret456"
-	t.Setenv("AWS_ACCESS_KEY_ID", accessKey)
-	t.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
+	os.Setenv("AWS_ACCESS_KEY_ID", accessKey)
+	os.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
+	defer os.Unsetenv("AWS_ACCESS_KEY_ID")
+	defer os.Unsetenv("AWS_SECRET_ACCESS_KEY")
 
 	// Create and start test cluster
 	cluster, err := startMiniCluster(t)

test/s3/normal/s3_integration_test.go

@@ -146,14 +146,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
 	}
 
-	// Set environment variables for admin credentials safely for this test
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
-		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
-	}
-	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
-	}
-
 	// Start weed mini in a goroutine by calling the command directly
 	cluster.wg.Add(1)
 	go func() {

test/s3/policy/policy_test.go

@@ -216,14 +216,6 @@ enabled = true
 	err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
 	require.NoError(t, err)
 
-	// Set environment variables for admin credentials safely for this test
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
-		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
-	}
-	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
-	}
-
 	cluster.wg.Add(1)
 	go func() {
 		defer cluster.wg.Done()

test/s3/spark/issue_8234_repro_test.go

@@ -1,76 +0,0 @@
-package spark
-
-import (
-	"strings"
-	"testing"
-)
-
-func setupSparkIssue8234Env(t *testing.T) *TestEnvironment {
-	t.Helper()
-
-	env := NewTestEnvironment()
-	if !env.dockerAvailable {
-		t.Skip("Docker not available, skipping Spark S3 integration test")
-	}
-
-	env.StartSeaweedFS(t)
-	t.Cleanup(func() { env.Cleanup(t) })
-
-	createObjectBucket(t, env, "test")
-	env.startSparkContainer(t)
-	return env
-}
-
-func TestSparkS3AppendIssue8234Regression(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping Spark integration test in short mode")
-	}
-
-	env := setupSparkIssue8234Env(t)
-
-	script := `
-import sys
-import pyspark.sql.types as T
-
-target = "s3a://test/test_table/"
-sc = spark.sparkContext
-df = sc.parallelize([1, 2, 3]).toDF(T.IntegerType())
-
-df.write.format("parquet").mode("append").save(target)
-print("FIRST_APPEND_OK")
-
-try:
-    df.write.format("parquet").mode("append").save(target)
-    print("SECOND_APPEND_OK")
-    spark.stop()
-    sys.exit(0)
-except Exception as err:
-    message = str(err)
-    print("SECOND_APPEND_FAILED")
-    print(message)
-    if "Copy Source must mention the source bucket and key" in message:
-        print("ISSUE_8234_REGRESSION")
-        spark.stop()
-        sys.exit(4)
-    spark.stop()
-    sys.exit(2)
-`
-
-	code, output := runSparkPyScript(t, env.sparkContainer, script, env.s3Port)
-	if code != 0 {
-		if strings.Contains(output, "ISSUE_8234_REGRESSION") {
-			t.Fatalf("issue #8234 regression detected; output:\n%s", output)
-		}
-		t.Fatalf("Spark script exited with code %d; output:\n%s", code, output)
-	}
-
-	if !strings.Contains(output, "FIRST_APPEND_OK") {
-		t.Fatalf("expected first append success marker in output, got:\n%s", output)
-	}
-	if !strings.Contains(output, "SECOND_APPEND_OK") {
-		t.Fatalf("expected second append success marker in output, got:\n%s", output)
-	}
-	if strings.Contains(output, "Copy Source must mention the source bucket and key") {
-		t.Fatalf("unexpected issue #8234 error in output:\n%s", output)
-	}
-}

test/s3/spark/issue_8285_repro_test.go

@@ -1,180 +0,0 @@
-package spark
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
-)
-
-func TestSparkS3TemporaryDirectoryCleanupIssue8285Regression(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping Spark integration test in short mode")
-	}
-
-	env := setupSparkIssue8234Env(t)
-
-	script := `
-import pyspark.sql.functions as F
-
-target = "s3a://test/issue-8285/output"
-
-spark.conf.set("spark.hadoop.fs.s3a.committer.name", "directory")
-spark.conf.set("spark.hadoop.fs.s3a.committer.magic.enabled", "false")
-spark.conf.set("spark.hadoop.fs.s3a.committer.staging.abort.pending.uploads", "true")
-spark.conf.set("spark.hadoop.fs.s3a.committer.staging.conflict-mode", "append")
-spark.conf.set("spark.hadoop.fs.s3a.committer.staging.tmp.path", "/tmp")
-spark.conf.set("spark.hadoop.fs.s3a.directory.marker.retention", "keep")
-
-df = spark.range(0, 200).repartition(12).withColumn("value", F.col("id") * 2)
-df.write.format("parquet").mode("overwrite").save(target)
-count = spark.read.parquet(target).count()
-print("WRITE_COUNT=" + str(count))
-`
-
-	code, output := runSparkPyScript(t, env.sparkContainer, script, env.s3Port)
-	if code != 0 {
-		t.Fatalf("Spark script exited with code %d; output:\n%s", code, output)
-	}
-	if !strings.Contains(output, "WRITE_COUNT=200") {
-		t.Fatalf("expected write/read success marker in output, got:\n%s", output)
-	}
-
-	keys := listObjectKeysByPrefix(t, env, "test", "issue-8285/")
-	var temporaryKeys []string
-	for _, key := range keys {
-		if hasTemporaryPathSegment(key) {
-			temporaryKeys = append(temporaryKeys, key)
-		}
-	}
-
-	if len(temporaryKeys) > 0 {
-		t.Fatalf("issue #8285 regression detected: found lingering _temporary artifacts: %v\nall keys: %v", temporaryKeys, keys)
-	}
-
-	temporaryCandidates := []string{
-		"issue-8285/output/_temporary/",
-		"issue-8285/output/_temporary/0/",
-		"issue-8285/output/_temporary/0/_temporary/",
-	}
-	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 35*time.Second)
-	if len(lingering) > 0 {
-		t.Fatalf("issue #8285 regression detected: lingering temporary directories: %v", lingering)
-	}
-}
-
-func listObjectKeysByPrefix(t *testing.T, env *TestEnvironment, bucketName, prefix string) []string {
-	t.Helper()
-	client := newS3Client(env)
-
-	pager := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
-		Bucket: aws.String(bucketName),
-		Prefix: aws.String(prefix),
-	})
-
-	var keys []string
-	for pager.HasMorePages() {
-		page, err := pager.NextPage(context.Background())
-		if err != nil {
-			t.Fatalf("failed listing objects for prefix %q: %v", prefix, err)
-		}
-		for _, object := range page.Contents {
-			keys = append(keys, aws.ToString(object.Key))
-		}
-	}
-
-	return keys
-}
-
-func headObjectInfo(t *testing.T, env *TestEnvironment, bucketName, key string) (bool, string, error) {
-	t.Helper()
-
-	client := newS3Client(env)
-	output, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(key),
-	})
-	if err == nil {
-		return true, aws.ToString(output.ContentType), nil
-	}
-
-	var notFound *s3types.NotFound
-	if strings.Contains(err.Error(), "NotFound") || strings.Contains(err.Error(), "NoSuchKey") || errors.As(err, &notFound) {
-		return false, "", nil
-	}
-	return false, "", err
-}
-
-func waitForObjectsToDisappear(t *testing.T, env *TestEnvironment, bucketName string, keys []string, timeout time.Duration) []string {
-	t.Helper()
-
-	deadline := time.Now().Add(timeout)
-	pending := make(map[string]struct{}, len(keys))
-	details := make(map[string]string, len(keys))
-	for _, key := range keys {
-		pending[key] = struct{}{}
-	}
-
-	for len(pending) > 0 && time.Now().Before(deadline) {
-		for key := range pending {
-			exists, contentType, err := headObjectInfo(t, env, bucketName, key)
-			if err != nil {
-				details[key] = fmt.Sprintf("%s (head_error=%v)", key, err)
-				continue
-			}
-			if !exists {
-				delete(pending, key)
-				delete(details, key)
-				continue
-			}
-			details[key] = fmt.Sprintf("%s (exists=true, contentType=%q)", key, contentType)
-		}
-		if len(pending) > 0 {
-			time.Sleep(2 * time.Second)
-		}
-	}
-
-	if len(pending) == 0 {
-		return nil
-	}
-
-	var lingering []string
-	for _, key := range keys {
-		if _, ok := pending[key]; !ok {
-			continue
-		}
-		if detail, hasDetail := details[key]; hasDetail {
-			lingering = append(lingering, detail)
-		} else {
-			lingering = append(lingering, key)
-		}
-	}
-	return lingering
-}
-
-func newS3Client(env *TestEnvironment) *s3.Client {
-	cfg := aws.Config{
-		Region:       "us-east-1",
-		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
-		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
-	}
-	return s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = true
-	})
-}
-
-func hasTemporaryPathSegment(key string) bool {
-	for _, segment := range strings.Split(strings.TrimSuffix(key, "/"), "/") {
-		if segment == "_temporary" {
-			return true
-		}
-	}
-	return false
-}

test/s3/spark/setup_test.go

@@ -1,349 +0,0 @@
-package spark
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math/rand"
-	"net"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
-	"github.com/testcontainers/testcontainers-go"
-)
-
-var (
-	miniProcessMu   sync.Mutex
-	lastMiniProcess *exec.Cmd
-)
-
-func stopPreviousMini() {
-	miniProcessMu.Lock()
-	defer miniProcessMu.Unlock()
-
-	if lastMiniProcess != nil && lastMiniProcess.Process != nil {
-		_ = lastMiniProcess.Process.Kill()
-		_ = lastMiniProcess.Wait()
-	}
-	lastMiniProcess = nil
-}
-
-func registerMiniProcess(cmd *exec.Cmd) {
-	miniProcessMu.Lock()
-	lastMiniProcess = cmd
-	miniProcessMu.Unlock()
-}
-
-func clearMiniProcess(cmd *exec.Cmd) {
-	miniProcessMu.Lock()
-	if lastMiniProcess == cmd {
-		lastMiniProcess = nil
-	}
-	miniProcessMu.Unlock()
-}
-
-type TestEnvironment struct {
-	dockerAvailable  bool
-	weedBinary       string
-	seaweedfsDataDir string
-	weedLogPath      string
-	weedLogFile      *os.File
-	masterPort       int
-	filerPort        int
-	s3Port           int
-	accessKey        string
-	secretKey        string
-	sparkContainer   testcontainers.Container
-	masterProcess    *exec.Cmd
-}
-
-func NewTestEnvironment() *TestEnvironment {
-	env := &TestEnvironment{
-		accessKey: "test",
-		secretKey: "test",
-	}
-
-	cmd := exec.Command("docker", "version")
-	env.dockerAvailable = cmd.Run() == nil
-
-	if weedPath, err := exec.LookPath("weed"); err == nil {
-		env.weedBinary = weedPath
-	}
-
-	return env
-}
-
-func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
-	t.Helper()
-
-	if env.weedBinary == "" {
-		t.Skip("weed binary not found in PATH, skipping Spark S3 integration test")
-	}
-
-	stopPreviousMini()
-
-	var err error
-	env.seaweedfsDataDir, err = os.MkdirTemp("", "seaweed-s3-spark-test-")
-	if err != nil {
-		t.Fatalf("failed to create temp directory: %v", err)
-	}
-
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-
-	bindIP := testutil.FindBindIP()
-	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
-	if err != nil {
-		t.Fatalf("failed to create IAM config: %v", err)
-	}
-
-	env.masterProcess = exec.Command(
-		env.weedBinary, "mini",
-		"-ip", bindIP,
-		"-ip.bind", "0.0.0.0",
-		"-master.port", fmt.Sprintf("%d", env.masterPort),
-		"-filer.port", fmt.Sprintf("%d", env.filerPort),
-		"-s3.port", fmt.Sprintf("%d", env.s3Port),
-		"-s3.config", iamConfigPath,
-		"-dir", env.seaweedfsDataDir,
-	)
-	weedLogPath := filepath.Join(env.seaweedfsDataDir, "weed-mini.log")
-	weedLogFile, err := os.Create(weedLogPath)
-	if err != nil {
-		t.Fatalf("failed to create weed log file: %v", err)
-	}
-	env.weedLogPath = weedLogPath
-	env.weedLogFile = weedLogFile
-	env.masterProcess.Stdout = weedLogFile
-	env.masterProcess.Stderr = weedLogFile
-	env.masterProcess.Env = append(os.Environ(),
-		"AWS_ACCESS_KEY_ID="+env.accessKey,
-		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
-	)
-
-	if err := env.masterProcess.Start(); err != nil {
-		t.Fatalf("failed to start weed mini: %v", err)
-	}
-	registerMiniProcess(env.masterProcess)
-
-	if !waitForPort(env.masterPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
-	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
-	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
-	}
-}
-
-func (env *TestEnvironment) startSparkContainer(t *testing.T) {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	defer cancel()
-
-	req := testcontainers.ContainerRequest{
-		Image:        "apache/spark:3.5.1",
-		ExposedPorts: []string{"4040/tcp"},
-		Env: map[string]string{
-			"SPARK_LOCAL_IP": "localhost",
-		},
-		ExtraHosts: []string{"host.docker.internal:host-gateway"},
-		Cmd:        []string{"/bin/sh", "-c", "sleep 7200"},
-	}
-
-	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
-		ContainerRequest: req,
-		Started:          true,
-	})
-	if err != nil {
-		t.Fatalf("failed to start spark container: %v", err)
-	}
-	env.sparkContainer = container
-}
-
-func (env *TestEnvironment) Cleanup(t *testing.T) {
-	if env.masterProcess != nil && env.masterProcess.Process != nil {
-		_ = env.masterProcess.Process.Kill()
-		_ = env.masterProcess.Wait()
-	}
-	clearMiniProcess(env.masterProcess)
-	if env.weedLogFile != nil {
-		_ = env.weedLogFile.Close()
-	}
-
-	if t.Failed() && os.Getenv("CI") != "" && env.weedLogPath != "" {
-		logData, err := os.ReadFile(env.weedLogPath)
-		if err != nil {
-			t.Logf("failed to read weed mini log file %s: %v", env.weedLogPath, err)
-		} else {
-			// Print the tail to keep CI output manageable while preserving failure context.
-			const maxTailBytes = 64 * 1024
-			start := 0
-			if len(logData) > maxTailBytes {
-				start = len(logData) - maxTailBytes
-			}
-			t.Logf("weed mini logs (tail, %d bytes):\n%s", len(logData)-start, string(logData[start:]))
-		}
-	}
-
-	if env.sparkContainer != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		_ = env.sparkContainer.Terminate(ctx)
-	}
-
-	if env.seaweedfsDataDir != "" {
-		_ = os.RemoveAll(env.seaweedfsDataDir)
-	}
-}
-
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		_ = listener.Close()
-
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		_ = grpcListener.Close()
-		return port
-	}
-
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			_ = conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
-func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
-	defer cancel()
-
-	pythonScript := fmt.Sprintf(`
-import glob
-import os
-import sys
-
-spark_home = os.environ.get("SPARK_HOME", "/opt/spark")
-python_path = os.path.join(spark_home, "python")
-py4j_glob = glob.glob(os.path.join(python_path, "lib", "py4j-*.zip"))
-ivy_dir = "/tmp/ivy"
-os.makedirs(ivy_dir, exist_ok=True)
-os.environ["AWS_REGION"] = "us-east-1"
-os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
-os.environ["AWS_ACCESS_KEY_ID"] = "test"
-os.environ["AWS_SECRET_ACCESS_KEY"] = "test"
-
-if python_path not in sys.path:
-    sys.path.insert(0, python_path)
-if py4j_glob and py4j_glob[0] not in sys.path:
-    sys.path.insert(0, py4j_glob[0])
-
-from pyspark.sql import SparkSession
-
-spark = (SparkSession.builder
-    .master("local[2]")
-    .appName("SeaweedFS S3 Spark Issue 8234 Repro")
-    .config("spark.sql.catalogImplementation", "hive")
-    .config("spark.executor.extraJavaOptions", "-Djdk.tls.client.protocols=TLSv1")
-    .config("spark.sql.parquet.int96RebaseModeInRead", "CORRECTED")
-    .config("spark.sql.parquet.int96RebaseModeInWrite", "CORRECTED")
-    .config("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED")
-    .config("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED")
-    .config("spark.sql.parquet.enableVectorizedReader", "false")
-    .config("spark.sql.parquet.mergeSchema", "false")
-    .config("spark.sql.parquet.writeLegacyFormat", "true")
-    .config("spark.sql.broadcastTimeout", "-1")
-    .config("spark.network.timeout", "600")
-    .config("spark.hadoop.hive.metastore.schema.verification", "false")
-    .config("spark.hadoop.hive.metastore.schema.verification.record.version", "false")
-    .config("spark.jars.ivy", ivy_dir)
-    .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:3.3.2,com.amazonaws:aws-java-sdk-bundle:1.12.262")
-    .config("spark.hadoop.fs.s3a.access.key", "test")
-    .config("spark.hadoop.fs.s3a.secret.key", "test")
-    .config("spark.hadoop.fs.s3a.endpoint", "host.docker.internal:%d")
-    .config("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version", "1")
-    .config("spark.hadoop.fs.s3a.path.style.access", "true")
-    .config("spark.hadoop.fs.s3a.fast.upload", "true")
-    .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false")
-    .config("spark.hadoop.fs.s3a.multiobjectdelete.enable", "true")
-    .config("spark.hadoop.fs.s3a.directory.marker.retention", "keep")
-    .config("spark.hadoop.fs.s3a.change.detection.version.required", "false")
-    .config("spark.hadoop.fs.s3a.change.detection.mode", "warn")
-    .config("spark.local.dir", "/tmp/spark-temp")
-    .getOrCreate())
-
-%s
-`, s3Port, script)
-
-	code, out, err := container.Exec(ctx, []string{"python3", "-c", pythonScript})
-	var output string
-	if out != nil {
-		outputBytes, readErr := io.ReadAll(out)
-		if readErr != nil {
-			output = fmt.Sprintf("failed to read container output: %v", readErr)
-		} else {
-			output = string(outputBytes)
-		}
-	}
-	if err != nil {
-		output = output + fmt.Sprintf("\ncontainer exec error: %v\n", err)
-	}
-	return code, output
-}
-
-func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
-	t.Helper()
-
-	cfg := aws.Config{
-		Region:       "us-east-1",
-		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
-		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
-	}
-
-	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = true
-	})
-
-	_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
-		Bucket: aws.String(bucketName),
-	})
-	if err != nil {
-		t.Fatalf("failed to create object bucket %s: %v", bucketName, err)
-	}
-}

test/s3tables/Makefile

@@ -1,321 +0,0 @@
-# Makefile for generating local S3Tables demo data via Trino and Spark.
-
-.PHONY: help check-deps check-services ensure-work-dir ensure-table-bucket \
-	populate-trino populate-spark populate clean
-
-# Local endpoints (host perspective)
-S3_ENDPOINT ?= http://localhost:8333
-CATALOG_ENDPOINT ?= http://localhost:8181
-
-# Container-visible endpoints (localhost -> host.docker.internal)
-S3_ENDPOINT_DOCKER ?= $(subst localhost,host.docker.internal,$(subst 127.0.0.1,host.docker.internal,$(S3_ENDPOINT)))
-CATALOG_ENDPOINT_DOCKER ?= $(subst localhost,host.docker.internal,$(subst 127.0.0.1,host.docker.internal,$(CATALOG_ENDPOINT)))
-
-# Auth and table-bucket settings
-# If credentials are not explicitly provided, try to reuse the ones from a running
-# local weed mini process (often started with inline AWS_ACCESS_KEY_ID/SECRET).
-WEED_RUNTIME_PID := $(shell pgrep -f "weed mini" 2>/dev/null | head -n 1)
-DETECTED_AWS_ACCESS_KEY_ID := $(shell if [ -n "$(WEED_RUNTIME_PID)" ]; then ps eww -p "$(WEED_RUNTIME_PID)" 2>/dev/null | sed -n 's/.*AWS_ACCESS_KEY_ID=\([^[:space:]]*\).*/\1/p'; fi)
-DETECTED_AWS_SECRET_ACCESS_KEY := $(shell if [ -n "$(WEED_RUNTIME_PID)" ]; then ps eww -p "$(WEED_RUNTIME_PID)" 2>/dev/null | sed -n 's/.*AWS_SECRET_ACCESS_KEY=\([^[:space:]]*\).*/\1/p'; fi)
-AWS_ACCESS_KEY_ID ?= $(if $(DETECTED_AWS_ACCESS_KEY_ID),$(DETECTED_AWS_ACCESS_KEY_ID),admin)
-AWS_SECRET_ACCESS_KEY ?= $(if $(DETECTED_AWS_SECRET_ACCESS_KEY),$(DETECTED_AWS_SECRET_ACCESS_KEY),admin)
-AWS_REGION ?= us-east-1
-TABLE_ACCOUNT_ID ?= admin
-TABLE_BUCKET ?= iceberg-tables
-WAREHOUSE ?= s3tablescatalog/$(TABLE_BUCKET)
-
-# Weed shell config for bucket bootstrap
-WEED_BIN ?= weed
-MASTER_ADDR ?= localhost:9333
-
-# Runtime images
-TRINO_IMAGE ?= trinodb/trino:479
-SPARK_IMAGE ?= tabulario/spark-iceberg:latest
-SPARK_PACKAGES ?= org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.2,org.apache.iceberg:iceberg-aws-bundle:1.7.2
-
-# Demo data layout
-TRINO_NAMESPACE ?= ui_trino
-TRINO_TABLE ?= customers
-SPARK_NAMESPACE ?= ui_spark
-SPARK_TABLE ?= events
-
-# Temp workspace for generated configs/sql
-WORK_DIR ?= /tmp/seaweedfs-s3tables-seed
-
-help: ## Show available targets and key variables
-	@echo "S3Tables local data generator"
-	@echo ""
-	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
-	@echo ""
-	@echo "Defaults:"
-	@echo "  S3_ENDPOINT=$(S3_ENDPOINT)"
-	@echo "  CATALOG_ENDPOINT=$(CATALOG_ENDPOINT)"
-	@echo "  TABLE_BUCKET=$(TABLE_BUCKET)"
-	@echo "  WAREHOUSE=$(WAREHOUSE)"
-
-check-deps: ## Verify local dependencies
-	@command -v docker >/dev/null 2>&1 || (echo "docker is required" && exit 1)
-	@if command -v $(WEED_BIN) >/dev/null 2>&1; then \
-		echo "Using weed binary: $(WEED_BIN)"; \
-	elif [ -x "$(WEED_BIN)" ]; then \
-		echo "Using weed binary path: $(WEED_BIN)"; \
-	else \
-		echo "weed binary not found at $(WEED_BIN)"; \
-		echo "Override with WEED_BIN=/path/to/weed"; \
-		exit 1; \
-	fi; \
-	if [ "$(origin AWS_ACCESS_KEY_ID)" = "file" ] && [ -n "$(DETECTED_AWS_ACCESS_KEY_ID)" ]; then \
-		echo "Using AWS access key from running weed mini process: $(DETECTED_AWS_ACCESS_KEY_ID)"; \
-	fi
-
-check-services: check-deps ## Validate local master/S3/catalog endpoints before seeding
-	@set -eu; \
-	master_addr="$(MASTER_ADDR)"; \
-	master_addr="$${master_addr%%,*}"; \
-	master_http=$$(printf '%s' "$$master_addr" | sed -E 's/^([^:]+:[0-9]+)\.[0-9]+$$/\1/'); \
-	echo "check-services: probing master status at http://$$master_http/cluster/status"; \
-	status_json=$$(curl -fsS --max-time 3 "http://$$master_http/cluster/status") || { \
-		echo "ERROR: cannot reach master status endpoint at http://$$master_http/cluster/status"; \
-		echo "Set MASTER_ADDR to a reachable master, e.g. MASTER_ADDR=localhost:9333"; \
-		exit 1; \
-	}; \
-	leader_addr=$$(printf '%s' "$$status_json" | sed -n 's/.*"Leader":"\([^"]*\)".*/\1/p'); \
-	if [ -n "$$leader_addr" ]; then \
-		leader_http=$$(printf '%s' "$$leader_addr" | sed -E 's/^([^:]+:[0-9]+)\.[0-9]+$$/\1/'); \
-		echo "check-services: master reports leader $$leader_addr (http: $$leader_http)"; \
-		if ! curl -fsS --max-time 3 "http://$$leader_http/cluster/status" >/dev/null; then \
-			echo "ERROR: master advertises leader $$leader_addr (http: $$leader_http), but it is unreachable from this host."; \
-			echo "This causes 'weed shell' (and make populate targets) to block waiting for master connection."; \
-			echo "Fix master advertised address (-ip / -ip.bind) or point MASTER_ADDR to a reachable leader."; \
-			exit 1; \
-		fi; \
-	fi; \
-	echo "check-services: probing S3 endpoint $(S3_ENDPOINT)"; \
-	s3_hostport=$$(printf '%s' "$(S3_ENDPOINT)" | sed -E 's#^https?://([^/]+)/?.*$$#\1#'); \
-	s3_host=$${s3_hostport%%:*}; \
-	s3_port=$${s3_hostport##*:}; \
-	if [ "$$s3_host" = "$$s3_port" ]; then \
-		s3_port=80; \
-	fi; \
-	if command -v nc >/dev/null 2>&1; then \
-		nc -z -w 3 "$$s3_host" "$$s3_port" || { \
-			echo "ERROR: S3 endpoint $(S3_ENDPOINT) is not reachable"; \
-			exit 1; \
-		}; \
-	else \
-		curl -sS --max-time 3 -o /dev/null "$(S3_ENDPOINT)" || { \
-			echo "ERROR: S3 endpoint $(S3_ENDPOINT) is not reachable"; \
-			exit 1; \
-		}; \
-	fi; \
-	echo "check-services: probing Iceberg catalog endpoint $(CATALOG_ENDPOINT)/v1/config"; \
-	curl -sS --max-time 3 -o /dev/null "$(CATALOG_ENDPOINT)/v1/config" || { \
-		echo "ERROR: Iceberg catalog endpoint $(CATALOG_ENDPOINT)/v1/config is not responding"; \
-		exit 1; \
-	}; \
-	echo "check-services: probing Iceberg auth with access key $(AWS_ACCESS_KEY_ID)"; \
-	auth_body=$$(mktemp -t s3tables-auth.XXXXXX); \
-	auth_code=$$(curl -sS --max-time 5 \
-		--aws-sigv4 "aws:amz:$(AWS_REGION):s3tables" \
-		--user "$(AWS_ACCESS_KEY_ID):$(AWS_SECRET_ACCESS_KEY)" \
-		-o "$$auth_body" -w "%{http_code}" "$(CATALOG_ENDPOINT)/v1/namespaces" || true); \
-	if [ "$$auth_code" = "403" ] && grep -q "access key ID you provided does not exist" "$$auth_body"; then \
-		echo "ERROR: AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) is not recognized by the running Iceberg service."; \
-		if [ -n "$(DETECTED_AWS_ACCESS_KEY_ID)" ] && [ -n "$(DETECTED_AWS_SECRET_ACCESS_KEY)" ] && [ "$(AWS_ACCESS_KEY_ID)" != "$(DETECTED_AWS_ACCESS_KEY_ID)" ]; then \
-			echo "Detected running weed mini credentials via process env:"; \
-			echo "  AWS_ACCESS_KEY_ID=$(DETECTED_AWS_ACCESS_KEY_ID)"; \
-			echo "Retry with:"; \
-			echo "  make populate AWS_ACCESS_KEY_ID=$(DETECTED_AWS_ACCESS_KEY_ID) AWS_SECRET_ACCESS_KEY=<secret>"; \
-		fi; \
-		rm -f "$$auth_body"; \
-		exit 1; \
-	fi; \
-	if [ "$$auth_code" = "403" ] && grep -q "Access Denied" "$$auth_body"; then \
-		echo "ERROR: provided AWS credentials are valid but do not have permission for Iceberg REST operations."; \
-		rm -f "$$auth_body"; \
-		exit 1; \
-	fi; \
-	rm -f "$$auth_body"
-
-ensure-work-dir: ## Create temporary workspace for generated files
-	@mkdir -p "$(WORK_DIR)/trino/catalog"
-
-ensure-table-bucket: check-services ## Create table bucket if it does not exist
-	@set -eu; \
-	echo "ensure-table-bucket: checking table bucket $(TABLE_BUCKET) for account $(TABLE_ACCOUNT_ID)"; \
-	get_output=$$(printf "s3tables.bucket -get -name $(TABLE_BUCKET) -account $(TABLE_ACCOUNT_ID)\nexit\n" | "$(WEED_BIN)" shell -master="$(MASTER_ADDR)" 2>&1 || true); \
-	if printf '%s' "$$get_output" | grep -q "NoSuchBucket"; then \
-		echo "Creating table bucket: $(TABLE_BUCKET)"; \
-		create_output=$$(printf "s3tables.bucket -create -name $(TABLE_BUCKET) -account $(TABLE_ACCOUNT_ID)\nexit\n" | "$(WEED_BIN)" shell -master="$(MASTER_ADDR)" 2>&1 || true); \
-		if ! printf '%s' "$$create_output" | grep -q "ARN: arn:aws:s3tables:"; then \
-			echo "ERROR: failed to create table bucket $(TABLE_BUCKET)."; \
-			printf '%s\n' "$$create_output"; \
-			exit 1; \
-		fi; \
-		echo "Created table bucket: $(TABLE_BUCKET)"; \
-	elif printf '%s' "$$get_output" | grep -q "ARN: arn:aws:s3tables:"; then \
-		echo "Table bucket already exists: $(TABLE_BUCKET)"; \
-	else \
-		echo "ERROR: unable to verify table bucket $(TABLE_BUCKET)."; \
-		printf '%s\n' "$$get_output"; \
-		exit 1; \
-	fi
-
-populate-trino: ensure-table-bucket ensure-work-dir ## Populate sample data via Trino
-	@echo "populate-trino: writing Trino catalog and SQL seeds"
-	@printf '%s\n' \
-		'connector.name=iceberg' \
-		'iceberg.catalog.type=rest' \
-		'iceberg.rest-catalog.uri=$(CATALOG_ENDPOINT_DOCKER)' \
-		'iceberg.rest-catalog.warehouse=$(WAREHOUSE)' \
-		'iceberg.file-format=PARQUET' \
-		'iceberg.unique-table-location=true' \
-		'fs.native-s3.enabled=true' \
-		's3.endpoint=$(S3_ENDPOINT_DOCKER)' \
-		's3.path-style-access=true' \
-		's3.signer-type=AwsS3V4Signer' \
-		's3.aws-access-key=$(AWS_ACCESS_KEY_ID)' \
-		's3.aws-secret-key=$(AWS_SECRET_ACCESS_KEY)' \
-		's3.region=$(AWS_REGION)' \
-		'iceberg.rest-catalog.security=SIGV4' \
-		'iceberg.rest-catalog.signing-name=s3tables' \
-		'iceberg.rest-catalog.prefix=$(TABLE_BUCKET)' \
-		> "$(WORK_DIR)/trino/catalog/iceberg.properties"
-	@printf '%s\n' \
-		'CREATE SCHEMA IF NOT EXISTS iceberg.$(TRINO_NAMESPACE);' \
-		'CREATE TABLE IF NOT EXISTS iceberg.$(TRINO_NAMESPACE).$(TRINO_TABLE) (' \
-		'  customer_id INTEGER,' \
-		'  customer_name VARCHAR,' \
-		'  country VARCHAR,' \
-		'  signup_date DATE' \
-		') WITH (' \
-		'  format = '\''PARQUET'\'',' \
-		'  partitioning = ARRAY['\''country'\'']' \
-		');' \
-		'INSERT INTO iceberg.$(TRINO_NAMESPACE).$(TRINO_TABLE)' \
-		'SELECT *' \
-		'FROM (' \
-		'  VALUES' \
-		'    (1, '\''Amanda Olson'\'', '\''US'\'', DATE '\''2024-01-10'\''),' \
-		'    (2, '\''Leonard Eads'\'', '\''US'\'', DATE '\''2024-03-22'\''),' \
-		'    (3, '\''Debbie Ward'\'', '\''MX'\'', DATE '\''2025-07-15'\''),' \
-		'    (4, '\''Donald Holt'\'', '\''CA'\'', DATE '\''2025-11-02'\'')' \
-		') AS src (customer_id, customer_name, country, signup_date)' \
-		'WHERE NOT EXISTS (' \
-		'  SELECT 1' \
-		'  FROM iceberg.$(TRINO_NAMESPACE).$(TRINO_TABLE) dst' \
-		'  WHERE dst.customer_id = src.customer_id' \
-		');' \
-		'SELECT count(*) AS row_count FROM iceberg.$(TRINO_NAMESPACE).$(TRINO_TABLE);' \
-		'SELECT count(*) AS us_row_count FROM iceberg.$(TRINO_NAMESPACE).$(TRINO_TABLE) WHERE country = '\''US'\'';' \
-		'SELECT count(*) AS partition_count FROM iceberg.$(TRINO_NAMESPACE)."$(TRINO_TABLE)$$partitions";' \
-	> "$(WORK_DIR)/trino_seed.sql"
-	@echo "populate-trino: starting Trino docker run"
-	@set -eu; \
-	container_name="seaweedfs-s3tables-trino-seed-$$RANDOM-$$RANDOM"; \
-	echo "populate-trino: launching Trino server container $$container_name"; \
-	docker run -d --name "$$container_name" \
-		--add-host host.docker.internal:host-gateway \
-		-v "$(WORK_DIR)/trino/catalog:/etc/trino/catalog" \
-		-v "$(WORK_DIR):/work" \
-		-e AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)" \
-		-e AWS_SECRET_ACCESS_KEY="$(AWS_SECRET_ACCESS_KEY)" \
-		-e AWS_REGION="$(AWS_REGION)" \
-		"$(TRINO_IMAGE)" >/dev/null; \
-	trap 'docker rm -f "$$container_name" >/dev/null 2>&1 || true' EXIT INT TERM; \
-	echo "populate-trino: waiting for Trino server readiness"; \
-	ready=0; \
-	for i in $$(seq 1 90); do \
-		if [ "$$(docker inspect -f '{{.State.Running}}' "$$container_name" 2>/dev/null || echo false)" != "true" ]; then \
-			break; \
-		fi; \
-		if docker exec "$$container_name" trino --execute "SELECT 1" >/dev/null 2>&1; then \
-			ready=1; \
-			break; \
-		fi; \
-		sleep 1; \
-	done; \
-	if [ "$$ready" -ne 1 ]; then \
-		echo "ERROR: Trino server in container $$container_name did not become ready"; \
-		docker logs "$$container_name" | tail -n 200; \
-		exit 1; \
-	fi; \
-	echo "populate-trino: running SQL seed script"; \
-	if ! docker exec "$$container_name" trino --catalog iceberg --output-format CSV --file /work/trino_seed.sql; then \
-		echo "ERROR: Trino seed query failed. Recent container logs:"; \
-		docker logs "$$container_name" | tail -n 200; \
-		exit 1; \
-	fi
-
-populate-spark: ensure-table-bucket ensure-work-dir ## Populate sample data via Spark SQL
-	@printf '%s\n' \
-		'CREATE NAMESPACE IF NOT EXISTS iceberg.$(SPARK_NAMESPACE);' \
-		'CREATE TABLE IF NOT EXISTS iceberg.$(SPARK_NAMESPACE).$(SPARK_TABLE) (' \
-		'  event_id BIGINT,' \
-		'  user_id STRING,' \
-		'  event_type STRING,' \
-		'  event_day STRING' \
-		') USING iceberg PARTITIONED BY (event_day);' \
-		'INSERT INTO iceberg.$(SPARK_NAMESPACE).$(SPARK_TABLE)' \
-		'SELECT *' \
-		'FROM (' \
-		'  VALUES' \
-		'    (1, '\''u001'\'', '\''page_view'\'', '\''2026-02-01'\''),' \
-		'    (2, '\''u002'\'', '\''checkout'\'', '\''2026-02-02'\''),' \
-		'    (3, '\''u003'\'', '\''purchase'\'', '\''2026-02-03'\''),' \
-		'    (4, '\''u004'\'', '\''refund'\'', '\''2026-02-04'\'')' \
-		') AS src (event_id, user_id, event_type, event_day)' \
-		'WHERE NOT EXISTS (' \
-		'  SELECT 1' \
-		'  FROM iceberg.$(SPARK_NAMESPACE).$(SPARK_TABLE) dst' \
-		'  WHERE dst.event_id = src.event_id' \
-		');' \
-		'SELECT count(*) AS row_count FROM iceberg.$(SPARK_NAMESPACE).$(SPARK_TABLE);' \
-		'SELECT count(*) AS partition_day_rows FROM iceberg.$(SPARK_NAMESPACE).$(SPARK_TABLE) WHERE event_day = '\''2026-02-02'\'';' \
-		'SELECT event_day, count(*) AS rows_per_day FROM iceberg.$(SPARK_NAMESPACE).$(SPARK_TABLE) GROUP BY event_day ORDER BY event_day;' \
-	> "$(WORK_DIR)/spark_seed.sql"
-	@echo "populate-spark: starting Spark docker run"
-	@docker run --rm \
-		--entrypoint /bin/bash \
-		--add-host host.docker.internal:host-gateway \
-		-v "$(WORK_DIR):/work" \
-		-e AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)" \
-		-e AWS_SECRET_ACCESS_KEY="$(AWS_SECRET_ACCESS_KEY)" \
-		-e AWS_REGION="$(AWS_REGION)" \
-		"$(SPARK_IMAGE)" \
-		-lc '/opt/spark/bin/spark-sql \
-			--packages "$(SPARK_PACKAGES)" \
-			--conf "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions" \
-			--conf "spark.sql.defaultCatalog=iceberg" \
-			--conf "spark.sql.catalog.iceberg=org.apache.iceberg.spark.SparkCatalog" \
-			--conf "spark.sql.catalog.iceberg.type=rest" \
-			--conf "spark.sql.catalog.iceberg.uri=$(CATALOG_ENDPOINT_DOCKER)" \
-			--conf "spark.sql.catalog.iceberg.warehouse=$(WAREHOUSE)" \
-			--conf "spark.sql.catalog.iceberg.prefix=$(TABLE_BUCKET)" \
-			--conf "spark.sql.catalog.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO" \
-			--conf "spark.sql.catalog.iceberg.s3.endpoint=$(S3_ENDPOINT_DOCKER)" \
-			--conf "spark.sql.catalog.iceberg.s3.path-style-access=true" \
-			--conf "spark.sql.catalog.iceberg.s3.access-key-id=$(AWS_ACCESS_KEY_ID)" \
-			--conf "spark.sql.catalog.iceberg.s3.secret-access-key=$(AWS_SECRET_ACCESS_KEY)" \
-			--conf "spark.sql.catalog.iceberg.s3.region=$(AWS_REGION)" \
-			--conf "spark.sql.catalog.iceberg.rest.sigv4-enabled=true" \
-			--conf "spark.sql.catalog.iceberg.rest.signing-name=s3tables" \
-			--conf "spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkCatalog" \
-			--conf "spark.sql.catalog.spark_catalog.type=rest" \
-			--conf "spark.sql.catalog.spark_catalog.uri=$(CATALOG_ENDPOINT_DOCKER)" \
-			--conf "spark.sql.catalog.spark_catalog.warehouse=$(WAREHOUSE)" \
-			--conf "spark.sql.catalog.spark_catalog.prefix=$(TABLE_BUCKET)" \
-			--conf "spark.sql.catalog.spark_catalog.io-impl=org.apache.iceberg.aws.s3.S3FileIO" \
-			--conf "spark.sql.catalog.spark_catalog.s3.endpoint=$(S3_ENDPOINT_DOCKER)" \
-			--conf "spark.sql.catalog.spark_catalog.s3.path-style-access=true" \
-			--conf "spark.sql.catalog.spark_catalog.s3.access-key-id=$(AWS_ACCESS_KEY_ID)" \
-			--conf "spark.sql.catalog.spark_catalog.s3.secret-access-key=$(AWS_SECRET_ACCESS_KEY)" \
-			--conf "spark.sql.catalog.spark_catalog.s3.region=$(AWS_REGION)" \
-			--conf "spark.sql.catalog.spark_catalog.rest.sigv4-enabled=true" \
-			--conf "spark.sql.catalog.spark_catalog.rest.signing-name=s3tables" \
-			-f /work/spark_seed.sql'
-
-populate: populate-trino populate-spark ## Populate sample data through Trino and Spark
-
-clean: ## Remove generated temporary files
-	@rm -rf "$(WORK_DIR)"

test/s3tables/README.txt

@@ -1,11 +0,0 @@
-Populate data run:
-
-  - make -C test/s3tables help
-  - make -C test/s3tables populate-trino
-  - make -C test/s3tables populate-spark
-
-  Run:
-
-  - make -C test/s3tables populate
-  - If your account id differs, override: make -C test/s3tables populate
-    TABLE_ACCOUNT_ID=000000000000

test/s3tables/catalog/iceberg_catalog_test.go

@@ -3,9 +3,7 @@
 package catalog
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -43,15 +41,16 @@ func hasDocker() bool {
 	return cmd.Run() == nil
 }
 
-// getFreePort returns an available ephemeral port and its listener
-func getFreePort() (int, net.Listener, error) {
+// getFreePort returns an available ephemeral port
+func getFreePort() (int, error) {
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
-		return 0, nil, err
+		return 0, err
 	}
+	defer listener.Close()
 
 	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, listener, nil
+	return addr.Port, nil
 }
 
 // NewTestEnvironment creates a new test environment
@@ -90,67 +89,43 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	}
 
 	// Allocate free ephemeral ports for each service
-	var listeners []net.Listener
-	defer func() {
-		for _, l := range listeners {
-			l.Close()
-		}
-	}()
-
-	var l net.Listener
-	s3Port, l, err := getFreePort()
+	s3Port, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for S3: %v", err)
 	}
-	listeners = append(listeners, l)
-
-	icebergPort, l, err := getFreePort()
+	icebergPort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Iceberg: %v", err)
 	}
-	listeners = append(listeners, l)
-
-	s3GrpcPort, l, err := getFreePort()
+	s3GrpcPort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for S3 gRPC: %v", err)
 	}
-	listeners = append(listeners, l)
-
-	masterPort, l, err := getFreePort()
+	masterPort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Master: %v", err)
 	}
-	listeners = append(listeners, l)
-
-	masterGrpcPort, l, err := getFreePort()
+	masterGrpcPort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Master gRPC: %v", err)
 	}
-	listeners = append(listeners, l)
-
-	filerPort, l, err := getFreePort()
+	filerPort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Filer: %v", err)
 	}
-	listeners = append(listeners, l)
-
-	filerGrpcPort, l, err := getFreePort()
+	filerGrpcPort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Filer gRPC: %v", err)
 	}
-	listeners = append(listeners, l)
-
-	volumePort, l, err := getFreePort()
+	volumePort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Volume: %v", err)
 	}
-	listeners = append(listeners, l)
 
-	volumeGrpcPort, l, err := getFreePort()
+	volumeGrpcPort, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Volume gRPC: %v", err)
 	}
-	listeners = append(listeners, l)
 
 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
@@ -315,182 +290,6 @@ func TestIcebergNamespaces(t *testing.T) {
 	}
 }
 
-// TestStageCreateAndFinalizeFlow verifies staged create remains invisible until assert-create commit finalizes table creation.
-func TestStageCreateAndFinalizeFlow(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	env.StartSeaweedFS(t)
-	createTableBucket(t, env, "warehouse")
-
-	namespace := "stage_ns"
-	tableName := "orders"
-
-	status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
-		"namespace": []string{namespace},
-	})
-	if err != nil {
-		t.Fatalf("Create namespace request failed: %v", err)
-	}
-	if status != http.StatusOK && status != http.StatusConflict {
-		t.Fatalf("Create namespace status = %d, want 200 or 409", status)
-	}
-
-	status, badReqResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
-		"stage-create": true,
-	})
-	if err != nil {
-		t.Fatalf("Stage create missing-name request failed: %v", err)
-	}
-	if status != http.StatusBadRequest {
-		t.Fatalf("Stage create missing-name status = %d, want 400", status)
-	}
-	errorObj, _ := badReqResp["error"].(map[string]any)
-	if got := errorObj["type"]; got != "BadRequestException" {
-		t.Fatalf("error.type = %v, want BadRequestException", got)
-	}
-	msg, _ := errorObj["message"].(string)
-	if !strings.Contains(strings.ToLower(msg), "table name is required") {
-		t.Fatalf("error.message = %v, want it to include %q", errorObj["message"], "table name is required")
-	}
-
-	status, stageResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
-		"name":         tableName,
-		"stage-create": true,
-	})
-	if err != nil {
-		t.Fatalf("Stage create request failed: %v", err)
-	}
-	if status != http.StatusOK {
-		t.Fatalf("Stage create status = %d, want 200", status)
-	}
-	stageLocation, _ := stageResp["metadata-location"].(string)
-	if !strings.HasSuffix(stageLocation, "/metadata/v1.metadata.json") {
-		t.Fatalf("stage metadata-location = %q, want suffix /metadata/v1.metadata.json", stageLocation)
-	}
-
-	status, _, err = doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
-	if err != nil {
-		t.Fatalf("Load staged table request failed: %v", err)
-	}
-	if status != http.StatusNotFound {
-		t.Fatalf("Load staged table status = %d, want 404", status)
-	}
-
-	status, commitResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
-		"requirements": []map[string]any{
-			{"type": "assert-create"},
-		},
-		"updates": []any{},
-	})
-	if err != nil {
-		t.Fatalf("Finalize commit request failed: %v", err)
-	}
-	if status != http.StatusOK {
-		t.Fatalf("Finalize commit status = %d, want 200", status)
-	}
-	commitLocation, _ := commitResp["metadata-location"].(string)
-	if !strings.HasSuffix(commitLocation, "/metadata/v1.metadata.json") {
-		t.Fatalf("final metadata-location = %q, want suffix /metadata/v1.metadata.json", commitLocation)
-	}
-
-	status, loadResp, err := doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
-	if err != nil {
-		t.Fatalf("Load finalized table request failed: %v", err)
-	}
-	if status != http.StatusOK {
-		t.Fatalf("Load finalized table status = %d, want 200", status)
-	}
-	loadLocation, _ := loadResp["metadata-location"].(string)
-	if loadLocation != commitLocation {
-		t.Fatalf("loaded metadata-location = %q, want %q", loadLocation, commitLocation)
-	}
-}
-
-// TestCommitMissingTableWithoutAssertCreate ensures missing-table commits still require assert-create for creation.
-func TestCommitMissingTableWithoutAssertCreate(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	env.StartSeaweedFS(t)
-	createTableBucket(t, env, "warehouse")
-
-	namespace := "stage_missing_assert_ns"
-	tableName := "missing_table"
-
-	status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
-		"namespace": []string{namespace},
-	})
-	if err != nil {
-		t.Fatalf("Create namespace request failed: %v", err)
-	}
-	if status != http.StatusOK && status != http.StatusConflict {
-		t.Fatalf("Create namespace status = %d, want 200 or 409", status)
-	}
-
-	status, _, err = doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
-		"requirements": []any{},
-		"updates":      []any{},
-	})
-	if err != nil {
-		t.Fatalf("Commit missing table request failed: %v", err)
-	}
-	if status != http.StatusNotFound {
-		t.Fatalf("Commit missing table status = %d, want 404", status)
-	}
-}
-
-// doIcebergJSONRequest decodes JSON object responses used by catalog tests.
-func doIcebergJSONRequest(env *TestEnvironment, method, path string, payload any) (int, map[string]any, error) {
-	url := env.IcebergURL() + path
-
-	var bodyReader io.Reader
-	if payload != nil {
-		data, err := json.Marshal(payload)
-		if err != nil {
-			return 0, nil, err
-		}
-		bodyReader = bytes.NewReader(data)
-	}
-
-	req, err := http.NewRequest(method, url, bodyReader)
-	if err != nil {
-		return 0, nil, err
-	}
-	if payload != nil {
-		req.Header.Set("Content-Type", "application/json")
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return 0, nil, err
-	}
-	defer resp.Body.Close()
-
-	data, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return resp.StatusCode, nil, err
-	}
-
-	if len(data) == 0 {
-		return resp.StatusCode, nil, nil
-	}
-
-	var decoded map[string]any
-	if err := json.Unmarshal(data, &decoded); err != nil {
-		return resp.StatusCode, nil, fmt.Errorf("failed to decode %s %s response: %w body=%s", method, path, err, string(data))
-	}
-	return resp.StatusCode, decoded, nil
-}
-
 // createTableBucket creates a table bucket via the S3Tables REST API
 func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	t.Helper()
@@ -511,10 +310,8 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	}
 	defer resp.Body.Close()
 
-	body, _ := io.ReadAll(resp.Body)
-	t.Logf("Create table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
-
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
+		body, _ := io.ReadAll(resp.Body)
 		t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
 	}
 	t.Logf("Created table bucket %s", bucketName)

test/s3tables/catalog/pyiceberg_test.go

@@ -56,6 +56,8 @@ func TestPyIcebergRestCatalog(t *testing.T) {
 
 	cmd := exec.Command("docker", "run", "--rm",
 		"--add-host", "host.docker.internal:host-gateway",
+		"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", "test"),
+		"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", "test"),
 		"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
 		"-v", fmt.Sprintf("%s:/app:ro", testDir),
 		"iceberg-rest-test",
@@ -76,69 +78,3 @@ func TestPyIcebergRestCatalog(t *testing.T) {
 		t.Errorf("PyIceberg test failed: %v", err)
 	}
 }
-
-// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
-// This test uses the default admin credentials that SeaweedFS creates on startup.
-func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	if !env.dockerAvailable {
-		t.Skip("Docker not available, skipping PyIceberg integration test")
-	}
-
-	// Use default admin credentials
-	testAccessKey := "admin"
-	testSecretKey := "admin"
-
-	// Start SeaweedFS (it will use default admin credentials from environment if set)
-	env.StartSeaweedFS(t)
-
-	// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
-	bucketName := "pyiceberg-auth-test"
-	createTableBucket(t, env, bucketName)
-
-	// Build the test working directory path
-	testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
-
-	// Run PyIceberg test using Docker with authentication
-	catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
-	s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
-	warehouse := fmt.Sprintf("s3://%s/", bucketName)
-
-	// Build the test image first for faster repeated runs
-	buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
-	buildCmd.Dir = testDir
-	if out, err := buildCmd.CombinedOutput(); err != nil {
-		t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
-	}
-
-	cmd := exec.Command("docker", "run", "--rm",
-		"--add-host", "host.docker.internal:host-gateway",
-		"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
-		"-v", fmt.Sprintf("%s:/app:ro", testDir),
-		"iceberg-rest-test",
-		"python3", "/app/test_rest_catalog_auth.py",
-		"--catalog-url", catalogURL,
-		"--warehouse", warehouse,
-		"--prefix", bucketName,
-		"--access-key", testAccessKey,
-		"--secret-key", testSecretKey,
-	)
-	cmd.Dir = testDir
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-
-	t.Logf("Running PyIceberg REST catalog test with authentication...")
-	t.Logf("  Catalog URL: %s", catalogURL)
-	t.Logf("  Warehouse: %s", warehouse)
-	t.Logf("  Access Key: %s", testAccessKey)
-
-	if err := cmd.Run(); err != nil {
-		t.Errorf("PyIceberg authenticated test failed: %v", err)
-	}
-}

test/s3tables/catalog/pyiceberg_test_helpers.go

@@ -1,36 +0,0 @@
-package catalog
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"testing"
-)
-
-// verifyTableBucketMetadata verifies that a table bucket was created with proper metadata
-func verifyTableBucketMetadata(t *testing.T, env *TestEnvironment, bucketName string) {
-	t.Helper()
-
-	// Use S3Tables REST API to get the bucket
-	endpoint := fmt.Sprintf("http://localhost:%d/buckets/%s", env.s3Port, bucketName)
-
-	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
-	if err != nil {
-		t.Fatalf("Failed to create request: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		t.Fatalf("Failed to get table bucket %s: %v", bucketName, err)
-	}
-	defer resp.Body.Close()
-
-	body, _ := io.ReadAll(resp.Body)
-	t.Logf("Get table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
-
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("Failed to get table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
-	}
-	t.Logf("Verified table bucket %s exists with metadata", bucketName)
-}

test/s3tables/catalog/test_rest_catalog.py

@@ -201,7 +201,6 @@ def main():
                     "uri": args.catalog_url,
                     "warehouse": args.warehouse,
                     "prefix": args.prefix,
-                    "s3.anonymous": "true",  # Disable AWS request signing for unauthenticated access
                 }
             )
             print(f"Successfully connected to catalog on attempt {attempt + 1}")

test/s3tables/catalog/test_rest_catalog_auth.py

@@ -1,223 +0,0 @@
-#!/usr/bin/env python3
-"""
-Iceberg REST Catalog Compatibility Test for SeaweedFS (Authenticated)
-
-This script tests the Iceberg REST Catalog API compatibility with authentication.
-
-Usage:
-    python3 test_rest_catalog_auth.py --catalog-url http://localhost:8182 \\
-        --access-key admin --secret-key admin
-
-Requirements:
-    pip install pyiceberg[s3fs]
-"""
-
-import argparse
-import sys
-from pyiceberg.catalog import load_catalog
-from pyiceberg.schema import Schema
-from pyiceberg.types import (
-    IntegerType,
-    LongType,
-    StringType,
-    NestedField,
-)
-from pyiceberg.exceptions import (
-    NamespaceAlreadyExistsError,
-    NoSuchNamespaceError,
-    TableAlreadyExistsError,
-    NoSuchTableError,
-)
-
-
-def test_config_endpoint(catalog):
-    """Test that the catalog config endpoint returns valid configuration."""
-    print("Testing /v1/config endpoint...")
-    # The catalog is already loaded which means config endpoint worked
-    print("  /v1/config endpoint working")
-    return True
-
-
-def test_namespace_operations(catalog, prefix):
-    """Test namespace CRUD operations."""
-    print("Testing namespace operations...")
-    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
-    
-    # List initial namespaces
-    namespaces = catalog.list_namespaces()
-    print(f"  Initial namespaces: {namespaces}")
-    
-    # Create namespace
-    try:
-        catalog.create_namespace(namespace)
-        print(f"  Created namespace: {namespace}")
-    except NamespaceAlreadyExistsError:
-        print(f"  ! Namespace already exists: {namespace}")
-    
-    # List namespaces (should include our new one)
-    namespaces = catalog.list_namespaces()
-    if namespace in namespaces:
-        print("  Namespace appears in list")
-    else:
-        print(f"  Namespace not found in list: {namespaces}")
-        return False
-    
-    # Get namespace properties
-    try:
-        props = catalog.load_namespace_properties(namespace)
-        print(f"  Loaded namespace properties: {props}")
-    except NoSuchNamespaceError:
-        print(f"  Failed to load namespace properties")
-        return False
-    
-    return True
-
-
-def test_table_operations(catalog, prefix):
-    """Test table CRUD operations."""
-    print("Testing table operations...")
-    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
-    table_name = "auth_test_table"
-    table_id = namespace + (table_name,)
-    
-    # Define a simple schema
-    schema = Schema(
-        NestedField(field_id=1, name="id", field_type=LongType(), required=True),
-        NestedField(field_id=2, name="name", field_type=StringType(), required=False),
-        NestedField(field_id=3, name="age", field_type=IntegerType(), required=False),
-    )
-    
-    # Create table
-    try:
-        table = catalog.create_table(
-            identifier=table_id,
-            schema=schema,
-        )
-        print(f"  Created table: {table_id}")
-    except TableAlreadyExistsError:
-        print(f"  ! Table already exists: {table_id}")
-        _ = catalog.load_table(table_id)
-    
-    # List tables
-    tables = catalog.list_tables(namespace)
-    if table_name in [t[1] for t in tables]:
-        print("  Table appears in list")
-    else:
-        print(f"  Table not found in list: {tables}")
-        return False
-    
-    # Load table
-    try:
-        loaded_table = catalog.load_table(table_id)
-        print(f"  Loaded table: {loaded_table.name()}")
-        print(f"    Schema: {loaded_table.schema()}")
-        print(f"    Location: {loaded_table.location()}")
-    except NoSuchTableError:
-        print(f"  Failed to load table")
-        return False
-    
-    return True
-
-
-def test_cleanup(catalog, prefix):
-    """Test table and namespace deletion."""
-    print("Testing cleanup operations...")
-    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
-    table_id = namespace + ("auth_test_table",)
-    
-    # Drop table
-    try:
-        catalog.drop_table(table_id)
-        print(f"  Dropped table: {table_id}")
-    except NoSuchTableError:
-        print(f"  ! Table already deleted: {table_id}")
-    
-    # Drop namespace
-    try:
-        catalog.drop_namespace(namespace)
-        print(f"  Dropped namespace: {namespace}")
-    except NoSuchNamespaceError:
-        print(f"  ! Namespace already deleted: {namespace}")
-    except Exception as e:
-        print(f"  ? Namespace drop error (may be expected): {e}")
-    
-    return True
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Test Iceberg REST Catalog with authentication")
-    parser.add_argument("--catalog-url", required=True, help="Iceberg REST Catalog URL")
-    parser.add_argument("--warehouse", default="s3://iceberg-test/", help="Warehouse location")
-    parser.add_argument("--prefix", required=True, help="Table bucket prefix")
-    parser.add_argument("--access-key", required=True, help="AWS Access Key ID")
-    parser.add_argument("--secret-key", required=True, help="AWS Secret Access Key")
-    parser.add_argument("--skip-cleanup", action="store_true", help="Skip cleanup at the end")
-    args = parser.parse_args()
-    
-    print(f"Connecting to Iceberg REST Catalog at: {args.catalog_url}")
-    print(f"Warehouse: {args.warehouse}")
-    print(f"Prefix: {args.prefix}")
-    print(f"Using authenticated access with key: {args.access_key}")
-    print()
-    
-    # Load the REST catalog with authentication
-    import time
-    max_retries = 10
-    catalog = None
-    for attempt in range(max_retries):
-        try:
-            catalog = load_catalog(
-                "rest",
-                **{
-                    "type": "rest",
-                    "uri": args.catalog_url,
-                    "warehouse": args.warehouse,
-                    "prefix": args.prefix,
-                    "s3.access-key-id": args.access_key,
-                    "s3.secret-access-key": args.secret_key,
-                }
-            )
-            print(f"Successfully connected to catalog on attempt {attempt + 1}")
-            break
-        except Exception as e:
-            if attempt < max_retries - 1:
-                print(f"  Attempt {attempt + 1} failed, retrying in 2s... ({e})")
-                time.sleep(2)
-            else:
-                print(f"  All {max_retries} attempts failed.")
-                raise e
-    
-    # Run tests
-    tests = [
-        ("Config Endpoint", lambda: test_config_endpoint(catalog)),
-        ("Namespace Operations", lambda: test_namespace_operations(catalog, args.prefix)),
-        ("Table Operations", lambda: test_table_operations(catalog, args.prefix)),
-    ]
-    
-    if not args.skip_cleanup:
-        tests.append(("Cleanup", lambda: test_cleanup(catalog, args.prefix)))
-    
-    passed = 0
-    failed = 0
-    
-    for name, test_fn in tests:
-        print(f"\n{'='*50}")
-        try:
-            if test_fn():
-                passed += 1
-                print(f"PASSED: {name}")
-            else:
-                failed += 1
-                print(f"FAILED: {name}")
-        except Exception as e:
-            failed += 1
-            print(f"ERROR in {name}: {e}")
-    
-    print(f"\n{'='*50}")
-    print(f"Results: {passed} passed, {failed} failed")
-    
-    return 0 if failed == 0 else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

test/s3tables/catalog_risingwave/risingwave_catalog_test.go

@@ -1,79 +0,0 @@
-package catalog_risingwave
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-)
-
-func TestRisingWaveIcebergCatalog(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	if !env.dockerAvailable {
-		t.Skip("Docker not available, skipping RisingWave integration test")
-	}
-
-	t.Log(">>> Starting SeaweedFS...")
-	env.StartSeaweedFS(t)
-	t.Log(">>> SeaweedFS started.")
-
-	tableBucket := "iceberg-tables"
-	t.Logf(">>> Creating table bucket: %s", tableBucket)
-	createTableBucket(t, env, tableBucket)
-
-	t.Log(">>> Starting RisingWave...")
-	env.StartRisingWave(t)
-	t.Log(">>> RisingWave started.")
-
-	// Create Iceberg namespace
-	createIcebergNamespace(t, env, "default")
-
-	// Create a catalog in RisingWave that points to SeaweedFS Iceberg REST API
-	icebergUri := env.dockerIcebergEndpoint()
-	s3Endpoint := env.dockerS3Endpoint()
-
-	tableName := "test_table_" + randomString(6)
-	createIcebergTable(t, env, tableBucket, "default", tableName)
-
-	sourceName := "test_source_" + randomString(6)
-	createSourceSql := fmt.Sprintf(`
-CREATE SOURCE %s WITH (
-    connector = 'iceberg',
-    catalog.type = 'rest',
-    catalog.uri = '%s',
-    catalog.name = 'default',
-    database.name = 'default',
-    table.name = '%s',
-    warehouse.path = 's3://%s',
-    s3.endpoint = '%s',
-    s3.region = 'us-east-1',
-    s3.access.key = '%s',
-    s3.secret.key = '%s',
-    s3.path.style.access = 'true',
-    catalog.rest.sigv4_enabled = 'true',
-    catalog.rest.signing_region = 'us-east-1',
-    catalog.rest.signing_name = 's3'
-);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
-
-	t.Logf(">>> Creating source %s...", sourceName)
-	runRisingWaveSQL(t, env.postgresSidecar, createSourceSql)
-
-	showSourcesOutput := runRisingWaveSQL(t, env.postgresSidecar, "SHOW SOURCES;")
-	if !strings.Contains(showSourcesOutput, sourceName) {
-		t.Fatalf("Expected source %s in SHOW SOURCES output:\n%s", sourceName, showSourcesOutput)
-	}
-
-	describeOutput := runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("DESCRIBE %s;", sourceName))
-	if !strings.Contains(describeOutput, "id") || !strings.Contains(describeOutput, "name") {
-		t.Fatalf("Expected id/name columns in DESCRIBE output:\n%s", describeOutput)
-	}
-
-	runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s LIMIT 0;", sourceName))
-
-	t.Log(">>> RisingWave Iceberg Catalog test passed!")
-}

test/s3tables/catalog_risingwave/risingwave_dml_test.go

@@ -1,219 +0,0 @@
-package catalog_risingwave
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestRisingWaveIcebergDML(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	if !env.dockerAvailable {
-		t.Skip("Docker not available, skipping RisingWave integration test")
-	}
-
-	t.Log(">>> Starting SeaweedFS...")
-	env.StartSeaweedFS(t)
-	t.Log(">>> SeaweedFS started.")
-
-	tableBucket := "iceberg-tables"
-	t.Logf(">>> Creating table bucket: %s", tableBucket)
-	createTableBucket(t, env, tableBucket)
-
-	t.Log(">>> Starting RisingWave...")
-	env.StartRisingWave(t)
-	t.Log(">>> RisingWave started.")
-
-	// Create Iceberg namespace
-	createIcebergNamespace(t, env, "default")
-
-	icebergUri := env.dockerIcebergEndpoint()
-	s3Endpoint := env.dockerS3Endpoint()
-
-	// 1. Test INSERT (Append-only)
-	t.Run("TestInsert", func(t *testing.T) {
-		tableName := "test_insert_" + randomString(6)
-		createIcebergTable(t, env, tableBucket, "default", tableName)
-
-		rwTableName := "rw_insert_" + randomString(6)
-		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("CREATE TABLE %s (id int, name varchar);", rwTableName))
-
-		sinkName := "test_sink_insert_" + randomString(6)
-		createSinkSql := fmt.Sprintf(`
-CREATE SINK %s FROM %s
-WITH (
-    connector = 'iceberg',
-    catalog.type = 'rest',
-    catalog.uri = '%s',
-    catalog.name = 'default',
-    database.name = 'default',
-    table.name = '%s',
-    warehouse.path = 's3://%s',
-    s3.endpoint = '%s',
-    s3.region = 'us-east-1',
-    s3.access.key = '%s',
-    s3.secret.key = '%s',
-    s3.path.style.access = 'true',
-    catalog.rest.sigv4_enabled = 'true',
-    catalog.rest.signing_region = 'us-east-1',
-    catalog.rest.signing_name = 's3',
-    type = 'append-only',
-    force_append_only = 'true'
-);`, sinkName, rwTableName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
-
-		t.Logf(">>> Creating sink %s...", sinkName)
-		runRisingWaveSQL(t, env.postgresSidecar, createSinkSql)
-
-		t.Log(">>> Inserting into RisingWave table...")
-		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob');", rwTableName))
-		runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;")
-
-		// Verify with Source
-		sourceName := "test_source_insert_" + randomString(6)
-		createSourceSql := fmt.Sprintf(`
-CREATE SOURCE %s WITH (
-    connector = 'iceberg',
-    catalog.type = 'rest',
-    catalog.uri = '%s',
-    catalog.name = 'default',
-    database.name = 'default',
-    table.name = '%s',
-    warehouse.path = 's3://%s',
-    s3.endpoint = '%s',
-    s3.region = 'us-east-1',
-    s3.access.key = '%s',
-    s3.secret.key = '%s',
-    s3.path.style.access = 'true',
-    catalog.rest.sigv4_enabled = 'true',
-    catalog.rest.signing_region = 'us-east-1',
-    catalog.rest.signing_name = 's3'
-);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
-
-		runRisingWaveSQL(t, env.postgresSidecar, createSourceSql)
-
-		t.Log(">>> Selecting from source to verify INSERT...")
-		verifyQuery(t, env, sourceName, "1 | Alice", "2 | Bob")
-	})
-
-	// 2. Test UPSERT (Update/Delete)
-	t.Run("TestUpsert", func(t *testing.T) {
-		tableName := "test_upsert_" + randomString(6)
-		// We need a table with PK for upsert to work effectively in RW logic,
-		// effectively maps to Iceberg v2 table.
-		createIcebergTable(t, env, tableBucket, "default", tableName)
-
-		rwTableName := "rw_upsert_" + randomString(6)
-		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("CREATE TABLE %s (id int PRIMARY KEY, name varchar);", rwTableName))
-
-		sinkName := "test_sink_upsert_" + randomString(6)
-		createSinkSql := fmt.Sprintf(`
-CREATE SINK %s FROM %s
-WITH (
-    connector = 'iceberg',
-    catalog.type = 'rest',
-    catalog.uri = '%s',
-    catalog.name = 'default',
-    database.name = 'default',
-    table.name = '%s',
-    warehouse.path = 's3://%s',
-    s3.endpoint = '%s',
-    s3.region = 'us-east-1',
-    s3.access.key = '%s',
-    s3.secret.key = '%s',
-    s3.path.style.access = 'true',
-    catalog.rest.sigv4_enabled = 'true',
-    catalog.rest.signing_region = 'us-east-1',
-    catalog.rest.signing_name = 's3',
-    type = 'upsert', -- Upsert mode
-    primary_key = 'id'
-);`, sinkName, rwTableName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
-
-		t.Logf(">>> Creating upsert sink %s...", sinkName)
-		runRisingWaveSQL(t, env.postgresSidecar, createSinkSql)
-
-		t.Log(">>> Inserting initial data...")
-		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("INSERT INTO %s VALUES (1, 'Charlie'), (2, 'Dave');", rwTableName))
-		runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;")
-
-		// Update 1, Delete 2
-		t.Log(">>> Updating and Deleting data...")
-		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("UPDATE %s SET name = 'Charles' WHERE id = 1;", rwTableName))
-		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("DELETE FROM %s WHERE id = 2;", rwTableName))
-		runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;")
-
-		// Verify with Source
-		sourceName := "test_source_upsert_" + randomString(6)
-		createSourceSql := fmt.Sprintf(`
-CREATE SOURCE %s WITH (
-    connector = 'iceberg',
-    catalog.type = 'rest',
-    catalog.uri = '%s',
-    catalog.name = 'default',
-    database.name = 'default',
-    table.name = '%s',
-    warehouse.path = 's3://%s',
-    s3.endpoint = '%s',
-    s3.region = 'us-east-1',
-    s3.access.key = '%s',
-    s3.secret.key = '%s',
-    s3.path.style.access = 'true',
-    catalog.rest.sigv4_enabled = 'true',
-    catalog.rest.signing_region = 'us-east-1',
-    catalog.rest.signing_name = 's3'
-);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
-
-		runRisingWaveSQL(t, env.postgresSidecar, createSourceSql)
-
-		t.Log(">>> Selecting from source to verify UPSERT...")
-		// Should see (1, 'Charles') and NOT (2, 'Dave')
-		verifyQuery(t, env, sourceName, "1 | Charles")
-		verifyQueryAbsence(t, env, sourceName, "2 | Dave")
-	})
-}
-
-func verifyQuery(t *testing.T, env *TestEnvironment, sourceName string, expectedSubstrings ...string) {
-	t.Helper()
-	var output string
-	for i := 0; i < 15; i++ {
-		output = runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s ORDER BY id;", sourceName))
-		allFound := true
-		for _, s := range expectedSubstrings {
-			if !strings.Contains(output, s) {
-				allFound = false
-				break
-			}
-		}
-		if allFound {
-			return
-		}
-		time.Sleep(2 * time.Second)
-	}
-	t.Fatalf("Failed to find expected data %v in output:\n%s", expectedSubstrings, output)
-}
-
-func verifyQueryAbsence(t *testing.T, env *TestEnvironment, sourceName string, unexpectedSubstrings ...string) {
-	t.Helper()
-	var output string
-	for i := 0; i < 15; i++ {
-		output = runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s ORDER BY id;", sourceName))
-		noneFound := true
-		for _, s := range unexpectedSubstrings {
-			if strings.Contains(output, s) {
-				noneFound = false
-				break
-			}
-		}
-		if noneFound {
-			return
-		}
-		time.Sleep(2 * time.Second)
-	}
-	t.Fatalf("Found unexpected data %v in output:\n%s", unexpectedSubstrings, output)
-}

test/s3tables/catalog_risingwave/setup_test.go

@@ -1,481 +0,0 @@
-package catalog_risingwave
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"math/rand"
-	"net"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
-)
-
-var (
-	miniProcessMu   sync.Mutex
-	lastMiniProcess *exec.Cmd
-)
-
-func stopPreviousMini() {
-	miniProcessMu.Lock()
-	defer miniProcessMu.Unlock()
-
-	if lastMiniProcess != nil && lastMiniProcess.Process != nil {
-		_ = lastMiniProcess.Process.Kill()
-		_ = lastMiniProcess.Wait()
-	}
-	lastMiniProcess = nil
-}
-
-func registerMiniProcess(cmd *exec.Cmd) {
-	miniProcessMu.Lock()
-	lastMiniProcess = cmd
-	miniProcessMu.Unlock()
-}
-
-func clearMiniProcess(cmd *exec.Cmd) {
-	miniProcessMu.Lock()
-	if lastMiniProcess == cmd {
-		lastMiniProcess = nil
-	}
-	miniProcessMu.Unlock()
-}
-
-type TestEnvironment struct {
-	t                   *testing.T
-	dockerAvailable     bool
-	seaweedfsDataDir    string
-	masterPort          int
-	filerPort           int
-	s3Port              int
-	icebergRestPort     int
-	risingwavePort      int
-	bindIP              string
-	accessKey           string
-	secretKey           string
-	risingwaveContainer string
-	postgresSidecar     string
-	masterProcess       *exec.Cmd
-	logFile             *os.File
-}
-
-func NewTestEnvironment(t *testing.T) *TestEnvironment {
-	env := &TestEnvironment{
-		t:         t,
-		accessKey: "AKIAIOSFODNN7EXAMPLE",
-		secretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
-	}
-
-	// Check if Docker is available
-	cmd := exec.Command("docker", "version")
-	env.dockerAvailable = cmd.Run() == nil
-
-	return env
-}
-
-func (env *TestEnvironment) hostMasterAddress() string {
-	return fmt.Sprintf("127.0.0.1:%d", env.masterPort)
-}
-
-func (env *TestEnvironment) hostS3Endpoint() string {
-	return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
-}
-
-func (env *TestEnvironment) hostIcebergEndpoint() string {
-	return fmt.Sprintf("http://127.0.0.1:%d", env.icebergRestPort)
-}
-
-func (env *TestEnvironment) dockerS3Endpoint() string {
-	return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
-}
-
-func (env *TestEnvironment) dockerIcebergEndpoint() string {
-	return fmt.Sprintf("http://host.docker.internal:%d", env.icebergRestPort)
-}
-
-func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
-	t.Helper()
-
-	stopPreviousMini()
-
-	var err error
-	env.seaweedfsDataDir, err = os.MkdirTemp("", "seaweed-risingwave-test-")
-	if err != nil {
-		t.Fatalf("failed to create temp directory: %v", err)
-	}
-
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
-	env.risingwavePort = mustFreePort(t, "RisingWave")
-
-	env.bindIP = testutil.FindBindIP()
-
-	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
-	if err != nil {
-		t.Fatalf("failed to create IAM config: %v", err)
-	}
-
-	// Create log file for SeaweedFS
-	logFile, err := os.Create(filepath.Join(env.seaweedfsDataDir, "seaweedfs.log"))
-	if err != nil {
-		t.Fatalf("failed to create log file: %v", err)
-	}
-	env.logFile = logFile
-
-	// Start SeaweedFS using weed mini (all-in-one including Iceberg REST)
-	env.masterProcess = exec.Command(
-		"weed", "mini",
-		"-ip", env.bindIP,
-		"-ip.bind", "0.0.0.0",
-		"-master.port", fmt.Sprintf("%d", env.masterPort),
-		"-filer.port", fmt.Sprintf("%d", env.filerPort),
-		"-s3.port", fmt.Sprintf("%d", env.s3Port),
-		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort),
-		"-s3.config", iamConfigPath,
-		"-dir", env.seaweedfsDataDir,
-	)
-	env.masterProcess.Stdout = logFile
-	env.masterProcess.Stderr = logFile
-	env.masterProcess.Env = append(os.Environ(),
-		"AWS_ACCESS_KEY_ID="+env.accessKey,
-		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
-		"ICEBERG_WAREHOUSE=s3://iceberg-tables",
-		"S3TABLES_DEFAULT_BUCKET=iceberg-tables",
-	)
-	if err := env.masterProcess.Start(); err != nil {
-		t.Fatalf("failed to start weed mini: %v", err)
-	}
-	registerMiniProcess(env.masterProcess)
-
-	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
-	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
-	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
-	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
-	}
-}
-
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-	minPort := 10000
-	maxPort := 55000 // Ensure port+10000 < 65535
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-	for i := 0; i < 1000; i++ {
-		port := minPort + r.Intn(maxPort-minPort)
-
-		// Check http port
-		ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
-		if err != nil {
-			continue
-		}
-		ln.Close()
-
-		// Check grpc port (weed mini uses port+10000)
-		ln2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000))
-		if err != nil {
-			continue
-		}
-		ln2.Close()
-
-		return port
-	}
-	t.Fatalf("failed to find a free port < %d for %s after 1000 attempts", maxPort, name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
-func (env *TestEnvironment) StartRisingWave(t *testing.T) {
-	t.Helper()
-
-	containerName := "seaweed-risingwave-" + randomString(8)
-	env.risingwaveContainer = containerName
-
-	cmd := exec.Command("docker", "run", "-d",
-		"--name", containerName,
-		"-p", fmt.Sprintf("%d:4566", env.risingwavePort),
-		"--add-host", "host.docker.internal:host-gateway",
-		"-e", "AWS_ACCESS_KEY_ID="+env.accessKey,
-		"-e", "AWS_SECRET_ACCESS_KEY="+env.secretKey,
-		"-e", "AWS_REGION=us-east-1",
-		"-e", "AWS_S3_PATH_STYLE_ACCESS=true",
-		"-e", "AWS_S3_FORCE_PATH_STYLE=true",
-		"risingwavelabs/risingwave:v2.5.0",
-		"playground",
-	)
-	if output, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("failed to start RisingWave container: %v\n%s", err, string(output))
-	}
-
-	// Start a sidecar postgres container for running psql commands
-	sidecarName := "seaweed-risingwave-sidecar-" + randomString(8)
-	env.postgresSidecar = sidecarName
-	sidecarCmd := exec.Command("docker", "run", "-d", "--rm",
-		"--name", sidecarName,
-		"--network", fmt.Sprintf("container:%s", containerName),
-		"postgres:16-alpine",
-		"sleep", "infinity",
-	)
-	if output, err := sidecarCmd.CombinedOutput(); err != nil {
-		t.Fatalf("failed to start postgres sidecar: %v\n%s", err, string(output))
-	}
-
-	// Wait for RisingWave port to be open on host
-	if !waitForPort(env.risingwavePort, 120*time.Second) {
-		t.Fatalf("timed out waiting for RisingWave port %d to be open", env.risingwavePort)
-	}
-
-	// Wait for RisingWave to be truly ready via psql in the sidecar.
-	if !env.waitForRisingWave(120 * time.Second) {
-		t.Fatalf("timed out waiting for RisingWave to be ready via psql")
-	}
-}
-
-func (env *TestEnvironment) waitForRisingWave(timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	env.t.Logf(">>> Waiting for RisingWave to be ready (timeout %v)...\n", timeout)
-	for time.Now().Before(deadline) {
-		if output, err := runPostgresClientSQL(env.postgresSidecar, "SELECT 1;"); err == nil {
-			env.t.Logf(">>> RisingWave is ready.\n")
-			return true
-		} else {
-			env.t.Logf(">>> RisingWave not ready yet: %v (Output: %s)\n", err, string(output))
-		}
-		time.Sleep(5 * time.Second)
-	}
-	return false
-}
-
-func runPostgresClientSQL(containerName, sql string) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	cmd := exec.CommandContext(ctx, "docker", "exec",
-		containerName,
-		"psql",
-		"-h", "127.0.0.1",
-		"-p", "4566",
-		"-U", "root",
-		"-d", "dev",
-		"-v", "ON_ERROR_STOP=1",
-		"-c", sql,
-	)
-	return cmd.CombinedOutput()
-}
-
-func (env *TestEnvironment) Cleanup(t *testing.T) {
-	t.Helper()
-
-	if env.risingwaveContainer != "" {
-		if t.Failed() {
-			logs, err := exec.Command("docker", "logs", env.risingwaveContainer).CombinedOutput()
-			if err == nil {
-				env.t.Logf(">>> RisingWave Logs:\n%s\n", string(logs))
-			} else {
-				env.t.Logf(">>> Failed to get RisingWave logs: %v\n", err)
-			}
-		}
-		_ = exec.Command("docker", "rm", "-f", env.risingwaveContainer).Run()
-	}
-
-	if env.postgresSidecar != "" {
-		_ = exec.Command("docker", "rm", "-f", env.postgresSidecar).Run()
-	}
-
-	if env.seaweedfsDataDir != "" && t.Failed() {
-		logPath := filepath.Join(env.seaweedfsDataDir, "seaweedfs.log")
-		if content, err := os.ReadFile(logPath); err == nil {
-			env.t.Logf(">>> SeaweedFS Logs:\n%s\n", string(content))
-		}
-		env.t.Logf(">>> Filer Contents:\n")
-		listFilerContents(t, env, "/")
-	}
-
-	if env.masterProcess != nil && env.masterProcess.Process != nil {
-		_ = env.masterProcess.Process.Kill()
-		_ = env.masterProcess.Wait()
-	}
-	clearMiniProcess(env.masterProcess)
-
-	if env.seaweedfsDataDir != "" {
-		if env.logFile != nil {
-			env.logFile.Close()
-		}
-		_ = os.RemoveAll(env.seaweedfsDataDir)
-	}
-}
-
-func runRisingWaveSQL(t *testing.T, containerName, sql string) string {
-	t.Helper()
-
-	output, err := runPostgresClientSQL(containerName, sql)
-	if err != nil {
-		t.Fatalf("RisingWave command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, string(output))
-	}
-	return string(output)
-}
-
-func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	cmd := exec.CommandContext(ctx, "weed", "shell",
-		fmt.Sprintf("-master=%s", env.hostMasterAddress()),
-	)
-	cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("failed to create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, string(output))
-	}
-}
-
-func doIcebergSignedJSONRequest(env *TestEnvironment, method, path string, payload any) (int, string, error) {
-	url := env.hostIcebergEndpoint() + path
-
-	var body io.Reader
-	var payloadHash string
-
-	if payload != nil {
-		data, err := json.Marshal(payload)
-		if err != nil {
-			return 0, "", err
-		}
-		body = bytes.NewReader(data)
-		// hash := sha256.Sum256(data)
-		// payloadHash = hex.EncodeToString(hash[:])
-		payloadHash = "UNSIGNED-PAYLOAD"
-	} else {
-		payloadHash = "UNSIGNED-PAYLOAD"
-	}
-
-	req, err := http.NewRequest(method, url, body)
-	if err != nil {
-		return 0, "", fmt.Errorf("failed to create request: %w", err)
-	}
-
-	if payload != nil {
-		req.Header.Set("Content-Type", "application/json")
-	}
-	req.Header.Set("X-Amz-Content-Sha256", payloadHash)
-
-	// Sign the request
-	credsProvider := credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")
-	creds, err := credsProvider.Retrieve(context.Background())
-	if err != nil {
-		return 0, "", fmt.Errorf("failed to retrieve credentials: %w", err)
-	}
-	signer := v4.NewSigner()
-
-	if err := signer.SignHTTP(context.Background(), creds, req, payloadHash, "s3", "us-east-1", time.Now()); err != nil {
-		return 0, "", fmt.Errorf("failed to sign request: %w", err)
-	}
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return 0, "", fmt.Errorf("request failed: %w", err)
-	}
-	defer resp.Body.Close()
-
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return 0, "", fmt.Errorf("failed to read response body: %w", err)
-	}
-
-	return resp.StatusCode, string(respBody), nil
-}
-
-func createIcebergNamespace(t *testing.T, env *TestEnvironment, namespace string) {
-	t.Helper()
-
-	status, raw, err := doIcebergSignedJSONRequest(env, "POST", "/v1/namespaces", map[string]any{
-		"namespace": []string{namespace},
-	})
-	if err != nil {
-		t.Fatalf("failed to create Iceberg namespace %s: %v", namespace, err)
-	}
-	if status != 200 && status != 409 {
-		t.Fatalf("failed to create Iceberg namespace %s: status %d body: %s", namespace, status, raw)
-	}
-}
-
-func createIcebergTable(t *testing.T, env *TestEnvironment, bucketName, namespace, tableName string) {
-	t.Helper()
-
-	createPath := fmt.Sprintf("/v1/namespaces/%s/tables", namespace)
-	status, raw, err := doIcebergSignedJSONRequest(env, "POST", createPath, map[string]any{
-		"name":     tableName,
-		"location": fmt.Sprintf("s3://%s/%s/%s", bucketName, namespace, tableName),
-		"schema": map[string]any{
-			"type": "struct",
-			"fields": []map[string]any{
-				{"id": 1, "name": "id", "required": false, "type": "int"},
-				{"id": 2, "name": "name", "required": false, "type": "string"},
-			},
-		},
-	})
-	if err != nil {
-		t.Fatalf("failed to create Iceberg table %s.%s in bucket %s: %v", namespace, tableName, bucketName, err)
-	}
-	if status != 200 && status != 409 {
-		t.Fatalf("failed to create Iceberg table %s.%s in bucket %s: status %d body: %s", namespace, tableName, bucketName, status, raw)
-	}
-}
-
-func listFilerContents(t *testing.T, env *TestEnvironment, path string) {
-	t.Helper()
-
-	cmd := exec.Command("weed", "shell",
-		fmt.Sprintf("-master=%s", env.hostMasterAddress()),
-	)
-	cmd.Stdin = strings.NewReader(fmt.Sprintf("fs.ls -R %s\nexit\n", path))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		env.t.Logf(">>> Warning: failed to list filer contents: %v\nOutput: %s\n", err, string(output))
-	} else {
-		env.t.Logf("%s\n", string(output))
-	}
-}
-
-func randomString(n int) string {
-	const letters = "abcdefghijklmnopqrstuvwxyz0123456789"
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letters[rand.Intn(len(letters))]
-	}
-	return string(b)
-}

test/s3tables/catalog_spark/setup_test.go

@@ -1,394 +0,0 @@
-package catalog_spark
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math/rand"
-	"net"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
-	"github.com/testcontainers/testcontainers-go"
-)
-
-var (
-	miniProcessMu   sync.Mutex
-	lastMiniProcess *exec.Cmd
-)
-
-func stopPreviousMini() {
-	miniProcessMu.Lock()
-	defer miniProcessMu.Unlock()
-
-	if lastMiniProcess != nil && lastMiniProcess.Process != nil {
-		_ = lastMiniProcess.Process.Kill()
-		_ = lastMiniProcess.Wait()
-	}
-	lastMiniProcess = nil
-}
-
-func registerMiniProcess(cmd *exec.Cmd) {
-	miniProcessMu.Lock()
-	lastMiniProcess = cmd
-	miniProcessMu.Unlock()
-}
-
-func clearMiniProcess(cmd *exec.Cmd) {
-	miniProcessMu.Lock()
-	if lastMiniProcess == cmd {
-		lastMiniProcess = nil
-	}
-	miniProcessMu.Unlock()
-}
-
-type TestEnvironment struct {
-	t                *testing.T
-	dockerAvailable  bool
-	seaweedfsDataDir string
-	sparkConfigDir   string
-	masterPort       int
-	filerPort        int
-	s3Port           int
-	icebergRestPort  int
-	accessKey        string
-	secretKey        string
-	sparkContainer   testcontainers.Container
-	masterProcess    *exec.Cmd
-}
-
-func NewTestEnvironment(t *testing.T) *TestEnvironment {
-	env := &TestEnvironment{
-		t:         t,
-		accessKey: "test",
-		secretKey: "test",
-	}
-
-	// Check if Docker is available
-	cmd := exec.Command("docker", "version")
-	env.dockerAvailable = cmd.Run() == nil
-
-	return env
-}
-
-func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
-	t.Helper()
-
-	stopPreviousMini()
-
-	var err error
-	env.seaweedfsDataDir, err = os.MkdirTemp("", "seaweed-spark-test-")
-	if err != nil {
-		t.Fatalf("failed to create temp directory: %v", err)
-	}
-
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
-
-	bindIP := testutil.FindBindIP()
-
-	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
-	if err != nil {
-		t.Fatalf("failed to create IAM config: %v", err)
-	}
-
-	// Start SeaweedFS using weed mini (all-in-one including Iceberg REST)
-	env.masterProcess = exec.Command(
-		"weed", "mini",
-		"-ip", bindIP,
-		"-ip.bind", "0.0.0.0",
-		"-master.port", fmt.Sprintf("%d", env.masterPort),
-		"-filer.port", fmt.Sprintf("%d", env.filerPort),
-		"-s3.port", fmt.Sprintf("%d", env.s3Port),
-		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort),
-		"-s3.config", iamConfigPath,
-		"-dir", env.seaweedfsDataDir,
-	)
-	env.masterProcess.Env = append(os.Environ(),
-		"AWS_ACCESS_KEY_ID="+env.accessKey,
-		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
-		"ICEBERG_WAREHOUSE=s3://iceberg-tables",
-		"S3TABLES_DEFAULT_BUCKET=iceberg-tables",
-	)
-	if err := env.masterProcess.Start(); err != nil {
-		t.Fatalf("failed to start weed mini: %v", err)
-	}
-	registerMiniProcess(env.masterProcess)
-
-	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
-	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
-	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
-	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
-	}
-}
-
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		listener.Close()
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		grpcListener.Close()
-		return port
-	}
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
-func (env *TestEnvironment) writeSparkConfig(t *testing.T, catalogBucket string) string {
-	t.Helper()
-
-	configDir, err := os.MkdirTemp("", "spark-config-")
-	if err != nil {
-		t.Fatalf("failed to create config directory: %v", err)
-	}
-
-	// Store for cleanup
-	env.sparkConfigDir = configDir
-
-	s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
-	catalogEndpoint := fmt.Sprintf("http://host.docker.internal:%d", env.icebergRestPort)
-
-	sparkConfig := fmt.Sprintf(`
-[spark]
-master = "local"
-app.name = "SeaweedFS Iceberg Test"
-
-[storage]
-s3.endpoint = "%s"
-s3.access-key = "test"
-s3.secret-key = "test"
-s3.path-style-access = "true"
-s3.bucket = "%s"
-
-[iceberg]
-catalog.type = "rest"
-catalog.uri = "%s"
-catalog.s3.endpoint = "%s"
-catalog.s3.access-key = "test"
-catalog.s3.secret-key = "test"
-catalog.s3.path-style-access = "true"
-`, s3Endpoint, catalogBucket, catalogEndpoint, s3Endpoint)
-
-	configPath := filepath.Join(configDir, "spark-config.ini")
-	if err := os.WriteFile(configPath, []byte(sparkConfig), 0644); err != nil {
-		t.Fatalf("failed to write spark config: %v", err)
-	}
-
-	return configDir
-}
-
-func (env *TestEnvironment) startSparkContainer(t *testing.T, configDir string) {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	defer cancel()
-
-	req := testcontainers.ContainerRequest{
-		Image:        "apache/spark:3.5.1",
-		ExposedPorts: []string{"4040/tcp"},
-		Mounts: testcontainers.Mounts(
-			testcontainers.BindMount(configDir, "/config"),
-		),
-		Env: map[string]string{
-			"SPARK_LOCAL_IP": "localhost",
-		},
-		ExtraHosts: []string{"host.docker.internal:host-gateway"},
-		Cmd:        []string{"/bin/sh", "-c", "sleep 3600"},
-	}
-
-	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
-		ContainerRequest: req,
-		Started:          true,
-	})
-	if err != nil {
-		t.Fatalf("failed to start spark container: %v", err)
-	}
-
-	env.sparkContainer = container
-}
-
-func (env *TestEnvironment) Cleanup(t *testing.T) {
-	t.Helper()
-
-	// Kill weed mini process
-	if env.masterProcess != nil && env.masterProcess.Process != nil {
-		env.masterProcess.Process.Kill()
-		env.masterProcess.Wait()
-	}
-	clearMiniProcess(env.masterProcess)
-
-	// Stop Spark container
-	if env.sparkContainer != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		env.sparkContainer.Terminate(ctx)
-	}
-
-	// Remove temporary directories after processes are stopped
-	if env.seaweedfsDataDir != "" {
-		os.RemoveAll(env.seaweedfsDataDir)
-	}
-	if env.sparkConfigDir != "" {
-		os.RemoveAll(env.sparkConfigDir)
-	}
-}
-
-func randomString(n int) string {
-	const letters = "abcdefghijklmnopqrstuvwxyz0123456789"
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letters[rand.Intn(len(letters))]
-	}
-	return string(b)
-}
-
-func runSparkPySQL(t *testing.T, container testcontainers.Container, sql string, icebergPort int, s3Port int) string {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
-	defer cancel()
-
-	pythonScript := fmt.Sprintf(`
-import glob
-import os
-import sys
-
-spark_home = os.environ.get("SPARK_HOME", "/opt/spark")
-python_path = os.path.join(spark_home, "python")
-py4j_glob = glob.glob(os.path.join(python_path, "lib", "py4j-*.zip"))
-ivy_dir = "/tmp/ivy"
-os.makedirs(ivy_dir, exist_ok=True)
-os.environ["AWS_REGION"] = "us-west-2"
-os.environ["AWS_DEFAULT_REGION"] = "us-west-2"
-os.environ["AWS_ACCESS_KEY_ID"] = "test"
-os.environ["AWS_SECRET_ACCESS_KEY"] = "test"
-if python_path not in sys.path:
-    sys.path.insert(0, python_path)
-if py4j_glob and py4j_glob[0] not in sys.path:
-    sys.path.insert(0, py4j_glob[0])
-
-from pyspark.sql import SparkSession
-
-spark = (SparkSession.builder
-    .appName("SeaweedFS Iceberg Test")
-    .config("spark.jars.ivy", ivy_dir)
-    .config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.2,org.apache.iceberg:iceberg-aws-bundle:1.5.2")
-    .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
-    .config("spark.sql.catalog.iceberg", "org.apache.iceberg.spark.SparkCatalog")
-    .config("spark.sql.catalog.iceberg.type", "rest")
-    .config("spark.sql.catalog.iceberg.metrics-reporter-impl", "org.apache.iceberg.metrics.LoggingMetricsReporter")
-    .config("spark.sql.catalog.iceberg.uri", "http://host.docker.internal:%d")
-    .config("spark.sql.catalog.iceberg.rest.auth.type", "sigv4")
-    .config("spark.sql.catalog.iceberg.rest.auth.sigv4.delegate-auth-type", "none")
-    .config("spark.sql.catalog.iceberg.rest.sigv4-enabled", "true")
-    .config("spark.sql.catalog.iceberg.rest.signing-name", "s3")
-    .config("spark.sql.catalog.iceberg.rest.access-key-id", "test")
-    .config("spark.sql.catalog.iceberg.rest.secret-access-key", "test")
-    .config("spark.sql.catalog.iceberg.s3.endpoint", "http://host.docker.internal:%d")
-    .config("spark.sql.catalog.iceberg.s3.region", "us-west-2")
-    .config("spark.sql.catalog.iceberg.s3.access-key", "test")
-    .config("spark.sql.catalog.iceberg.s3.secret-key", "test")
-    .config("spark.sql.catalog.iceberg.s3.path-style-access", "true")
-    .config("spark.sql.catalog.iceberg.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
-    .getOrCreate())
-
-%s
-`, icebergPort, s3Port, sql)
-
-	code, out, err := container.Exec(ctx, []string{"python3", "-c", pythonScript})
-	var output string
-	if out != nil {
-		outputBytes, readErr := io.ReadAll(out)
-		if readErr != nil {
-			t.Logf("failed to read output: %v", readErr)
-		} else {
-			output = string(outputBytes)
-		}
-	}
-	if code != 0 {
-		t.Logf("Spark Python execution failed with code %d: %v, output: %s", code, err, output)
-		return output
-	}
-
-	return output
-}
-
-func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
-	t.Helper()
-
-	masterGrpcPort := env.masterPort + 10000
-	cmd := exec.Command("weed", "shell",
-		fmt.Sprintf("-master=localhost:%d.%d", env.masterPort, masterGrpcPort),
-	)
-	cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("failed to create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, string(output))
-	}
-}
-
-func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
-	t.Helper()
-
-	cfg := aws.Config{
-		Region:       "us-east-1",
-		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
-		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
-	}
-
-	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = true
-	})
-
-	_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
-		Bucket: aws.String(bucketName),
-	})
-	if err != nil {
-		t.Fatalf("failed to create object bucket %s: %v", bucketName, err)
-	}
-}

test/s3tables/catalog_spark/spark_operations_test.go

@@ -1,279 +0,0 @@
-package catalog_spark
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/testcontainers/testcontainers-go"
-)
-
-// waitForSparkReady polls Spark to verify it's ready by executing a simple query
-func waitForSparkReady(t *testing.T, container testcontainers.Container, icebergPort int, s3Port int, timeout time.Duration) {
-	t.Helper()
-
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		output := runSparkPySQL(t, container, `
-spark.sql("SELECT 1 as test")
-print("Spark ready")
-`, icebergPort, s3Port)
-		if strings.Contains(output, "Spark ready") {
-			return
-		}
-		time.Sleep(5 * time.Second)
-	}
-	t.Fatalf("Spark did not become ready within %v", timeout)
-}
-
-// setupSparkTestEnv initializes a test environment with SeaweedFS and Spark containers
-func setupSparkTestEnv(t *testing.T) (*TestEnvironment, string, string) {
-	t.Helper()
-
-	env := NewTestEnvironment(t)
-
-	if !env.dockerAvailable {
-		t.Skip("Docker not available, skipping Spark integration test")
-	}
-
-	t.Logf(">>> Starting SeaweedFS...")
-	env.StartSeaweedFS(t)
-	t.Cleanup(func() { env.Cleanup(t) })
-
-	tableBucket := "iceberg-tables"
-	catalogBucket := tableBucket
-	createTableBucket(t, env, tableBucket)
-
-	configDir := env.writeSparkConfig(t, catalogBucket)
-	env.startSparkContainer(t, configDir)
-
-	// Poll for Spark readiness instead of fixed sleep
-	waitForSparkReady(t, env.sparkContainer, env.icebergRestPort, env.s3Port, 10*time.Minute)
-
-	return env, catalogBucket, tableBucket
-}
-
-// TestSparkCatalogBasicOperations tests basic Spark Iceberg catalog operations
-func TestSparkCatalogBasicOperations(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env, _, _ := setupSparkTestEnv(t)
-
-	// Test 1: Create a namespace (database)
-	t.Logf(">>> Test 1: Creating namespace")
-	namespace := "spark_test_" + randomString(6)
-	sparkSQL := fmt.Sprintf(`
-spark.sql("CREATE NAMESPACE iceberg.%s")
-print("Namespace created")
-`, namespace)
-	output := runSparkPySQL(t, env.sparkContainer, sparkSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Namespace created") {
-		t.Fatalf("namespace creation failed, output: %s", output)
-	}
-
-	// Test 2: Create a table
-	t.Logf(">>> Test 2: Creating table")
-	tableName := "test_table_" + randomString(6)
-	createTableSQL := fmt.Sprintf(`
-spark.sql("""
-CREATE TABLE iceberg.%s.%s (
-    id INT,
-    name STRING,
-    age INT
-)
-USING iceberg
-""")
-print("Table created")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, createTableSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Table created") {
-		t.Fatalf("table creation failed, output: %s", output)
-	}
-
-	// Test 3: Insert data
-	t.Logf(">>> Test 3: Inserting data")
-	insertDataSQL := fmt.Sprintf(`
-spark.sql("""
-INSERT INTO iceberg.%s.%s VALUES
-    (1, 'Alice', 30),
-    (2, 'Bob', 25),
-    (3, 'Charlie', 35)
-""")
-print("Data inserted")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, insertDataSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Data inserted") {
-		t.Fatalf("data insertion failed, output: %s", output)
-	}
-
-	// Test 4: Query data
-	t.Logf(">>> Test 4: Querying data")
-	querySQL := fmt.Sprintf(`
-result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
-result.show()
-count = result.collect()[0]['count']
-print(f"Row count: {count}")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, querySQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Row count: 3") {
-		t.Errorf("expected row count 3, got output: %s", output)
-	}
-
-	// Test 5: Update data
-	t.Logf(">>> Test 5: Updating data")
-	updateSQL := fmt.Sprintf(`
-spark.sql("""
-UPDATE iceberg.%s.%s SET age = 31 WHERE id = 1
-""")
-print("Data updated")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, updateSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Data updated") {
-		t.Errorf("data update failed, output: %s", output)
-	}
-
-	// Test 6: Delete data
-	t.Logf(">>> Test 6: Deleting data")
-	deleteSQL := fmt.Sprintf(`
-spark.sql("""
-DELETE FROM iceberg.%s.%s WHERE id = 3
-""")
-print("Data deleted")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, deleteSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Data deleted") {
-		t.Errorf("data delete failed, output: %s", output)
-	}
-
-	// Verify final count
-	t.Logf(">>> Verifying final data")
-	finalCountSQL := fmt.Sprintf(`
-result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
-result.show()
-count = result.collect()[0]['count']
-print(f"Final row count: {count}")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, finalCountSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Final row count: 2") {
-		t.Errorf("expected final row count 2, got output: %s", output)
-	}
-
-	t.Logf(">>> All tests passed")
-}
-
-// TestSparkTimeTravel tests Spark Iceberg time travel capabilities
-func TestSparkTimeTravel(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env, _, _ := setupSparkTestEnv(t)
-
-	namespace := "time_travel_test_" + randomString(6)
-	tableName := "tt_table_" + randomString(6)
-
-	// Create namespace and table
-	setupSQL := fmt.Sprintf(`
-spark.sql("CREATE NAMESPACE iceberg.%s")
-spark.sql("""
-CREATE TABLE iceberg.%s.%s (
-    id INT,
-    value INT
-)
-USING iceberg
-""")
-print("Setup complete")
-`, namespace, namespace, tableName)
-	output := runSparkPySQL(t, env.sparkContainer, setupSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Setup complete") {
-		t.Fatalf("setup failed for namespace %s and table %s, output: %s", namespace, tableName, output)
-	}
-
-	// Insert initial data
-	t.Logf(">>> Inserting initial data")
-	insertSQL := fmt.Sprintf(`
-import time
-from datetime import timedelta
-spark.sql("""
-INSERT INTO iceberg.%s.%s VALUES (1, 10)
-""")
-ts = None
-for _ in range(10):
-    try:
-        ts = spark.sql("SELECT committed_at FROM iceberg.%s.%s.snapshots ORDER BY committed_at DESC LIMIT 1").collect()[0]["committed_at"]
-        if ts is not None:
-            break
-    except Exception as e:
-        print(f"Snapshot query failed: {e}")
-    time.sleep(1)
-if ts is None:
-    raise RuntimeError("Failed to read snapshot committed_at")
-ts_for_time_travel = ts + timedelta(seconds=1)
-print(f"Snapshot timestamp: {ts_for_time_travel.strftime('%%Y-%%m-%%d %%H:%%M:%%S')}")
-`, namespace, tableName, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, insertSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Snapshot timestamp:") {
-		t.Fatalf("failed to get snapshot timestamp: %s", output)
-	}
-
-	// Extract snapshot timestamp from output - look specifically for the "Snapshot timestamp:" line
-	var snapshotTS string
-	tsRe := regexp.MustCompile(`Snapshot timestamp:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})`)
-	matches := tsRe.FindStringSubmatch(output)
-	if len(matches) > 1 {
-		snapshotTS = matches[1]
-	}
-
-	if snapshotTS == "" {
-		t.Fatalf("could not extract snapshot timestamp from output: %s", output)
-	}
-
-	// Wait to ensure the next insert gets a distinct snapshot timestamp
-	time.Sleep(2 * time.Second)
-
-	// Insert more data
-	t.Logf(">>> Inserting more data")
-	insertMoreSQL := fmt.Sprintf(`
-spark.sql("""
-INSERT INTO iceberg.%s.%s VALUES (2, 20)
-""")
-print("More data inserted")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, insertMoreSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "More data inserted") {
-		t.Fatalf("failed to insert more data, output: %s", output)
-	}
-
-	// Verify count increased to 2
-	t.Logf(">>> Verifying row count after second insert")
-	verifySQL := fmt.Sprintf(`
-result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
-count = result.collect()[0]['count']
-print(f"Current row count: {count}")
-`, namespace, tableName)
-	output = runSparkPySQL(t, env.sparkContainer, verifySQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Current row count: 2") {
-		t.Fatalf("expected current row count 2 after second insert, got output: %s", output)
-	}
-
-	// Time travel to first snapshot
-	t.Logf(">>> Time traveling to first snapshot")
-	timeTravelSQL := fmt.Sprintf(`
-result = spark.sql("""
-SELECT COUNT(*) as count FROM iceberg.%s.%s TIMESTAMP AS OF '%s'
-""")
-result.show()
-count = result.collect()[0]['count']
-print(f"Count at snapshot: {count}")
-`, namespace, tableName, snapshotTS)
-	output = runSparkPySQL(t, env.sparkContainer, timeTravelSQL, env.icebergRestPort, env.s3Port)
-	if !strings.Contains(output, "Count at snapshot: 1") {
-		t.Errorf("expected count 1 at first snapshot, got: %s", output)
-	}
-
-	t.Logf(">>> Time travel test passed")
-}

test/s3tables/catalog_trino/trino_blog_operations_test.go

@@ -1,248 +0,0 @@
-package catalog_trino
-
-import (
-	"fmt"
-	"os/exec"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestTrinoBlogOperations(t *testing.T) {
-	env := setupTrinoTest(t)
-	defer env.Cleanup(t)
-
-	schemaName := "blog_ns_" + randomString(6)
-	customersTable := "customers_" + randomString(6)
-	trinoCustomersTable := "trino_customers_" + randomString(6)
-	warehouseBucket := "iceberg-tables"
-	customersLocation := fmt.Sprintf("s3://%s/%s/%s_%s", warehouseBucket, schemaName, customersTable, randomString(6))
-	trinoCustomersLocation := fmt.Sprintf("s3://%s/%s/%s_%s", warehouseBucket, schemaName, trinoCustomersTable, randomString(6))
-
-	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
-	defer runTrinoSQLAllowNamespaceNotEmpty(t, env.trinoContainer, fmt.Sprintf("DROP SCHEMA IF EXISTS iceberg.%s CASCADE", schemaName))
-	defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, trinoCustomersTable))
-	defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, customersTable))
-	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, trinoCustomersTable))
-	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, customersTable))
-
-	createCustomersSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s (
-    customer_sk INT,
-    customer_id VARCHAR,
-    salutation VARCHAR,
-    first_name VARCHAR,
-    last_name VARCHAR,
-    preferred_cust_flag VARCHAR,
-    birth_day INT,
-    birth_month INT,
-    birth_year INT,
-    birth_country VARCHAR,
-    login VARCHAR
-) WITH (
-    format = 'PARQUET',
-    sorted_by = ARRAY['customer_id'],
-    location = '%s'
-)`, schemaName, customersTable, customersLocation)
-	runTrinoSQLAllowExists(t, env.trinoContainer, createCustomersSQL)
-
-	insertCustomersSQL := fmt.Sprintf(`INSERT INTO iceberg.%s.%s VALUES
-    (1, 'AAAAA', 'Mrs', 'Amanda', 'Olson', 'Y', 8, 4, 1984, 'US', 'aolson'),
-    (2, 'AAAAB', 'Mr',  'Leonard', 'Eads',  'N', 22, 6, 2001, 'US', 'leads'),
-    (3, 'BAAAA', 'Mr',  'David',   'White', 'Y', 16, 2, 1999, 'US', 'dwhite'),
-    (4, 'BBAAA', 'Mr',  'Melvin',  'Lee',   'N', 30, 3, 1973, 'US', 'mlee'),
-    (5, 'AACAA', 'Mr',  'Donald',  'Holt',  'N', 2,  6, 1982, 'CA', 'dholt'),
-    (6, 'ABAAA', 'Mrs', 'Jacqueline', 'Harvey', 'N', 5, 12, 1988, 'US', 'jharvey'),
-    (7, 'BBAAA', 'Ms',  'Debbie',  'Ward',  'N', 6,  1, 2006, 'MX', 'dward'),
-    (8, 'ACAAA', 'Mr',  'Tim',     'Strong', 'N', 15, 7, 1976, 'US', 'tstrong')
-`, schemaName, customersTable)
-	runTrinoSQL(t, env.trinoContainer, insertCustomersSQL)
-
-	countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, customersTable))
-	rowCount := mustParseCSVInt64(t, countOutput)
-	if rowCount != 8 {
-		t.Fatalf("expected 8 rows in customers table, got %d", rowCount)
-	}
-
-	output := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT first_name FROM iceberg.%s.%s WHERE customer_sk = 1", schemaName, customersTable))
-	if !strings.Contains(output, "Amanda") {
-		t.Fatalf("expected sample query to include Amanda, got: %s", output)
-	}
-
-	ctasSQL := fmt.Sprintf(`CREATE TABLE iceberg.%s.%s
-WITH (
-    format = 'PARQUET',
-    location = '%s'
-)
-AS SELECT * FROM iceberg.%s.%s`, schemaName, trinoCustomersTable, trinoCustomersLocation, schemaName, customersTable)
-	ctasInsertSQL := fmt.Sprintf("INSERT INTO iceberg.%s.%s SELECT * FROM iceberg.%s.%s", schemaName, trinoCustomersTable, schemaName, customersTable)
-	ctasDeleteSQL := fmt.Sprintf("DELETE FROM iceberg.%s.%s", schemaName, trinoCustomersTable)
-	runTrinoCTAS(t, env.trinoContainer, ctasSQL, ctasDeleteSQL, ctasInsertSQL)
-
-	countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable))
-	rowCount = mustParseCSVInt64(t, countOutput)
-	if rowCount != 8 {
-		t.Fatalf("expected 8 rows in CTAS table, got %d", rowCount)
-	}
-
-	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("ALTER TABLE iceberg.%s.%s ADD COLUMN updated_at TIMESTAMP", schemaName, trinoCustomersTable))
-	output = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DESCRIBE iceberg.%s.%s", schemaName, trinoCustomersTable))
-	if !strings.Contains(output, "updated_at") {
-		t.Fatalf("expected updated_at column in describe output, got: %s", output)
-	}
-
-	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("UPDATE iceberg.%s.%s SET updated_at = current_timestamp", schemaName, trinoCustomersTable))
-
-	// Sleep to ensure timestamps are in the past for time travel queries
-	time.Sleep(1 * time.Second)
-
-	snapshotOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT snapshot_id FROM iceberg.%s."%s$snapshots" ORDER BY committed_at DESC LIMIT 1`, schemaName, trinoCustomersTable))
-	snapshotID := mustParseCSVInt64(t, snapshotOutput)
-	if snapshotID == 0 {
-		t.Fatalf("expected snapshot ID from snapshots table, got 0")
-	}
-
-	filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, schemaName, trinoCustomersTable))
-	if !hasCSVDataRow(filesOutput) {
-		t.Fatalf("expected files metadata rows, got: %s", filesOutput)
-	}
-
-	historyOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT made_current_at FROM iceberg.%s."%s$history" LIMIT 1`, schemaName, trinoCustomersTable))
-	if !hasCSVDataRow(historyOutput) {
-		t.Fatalf("expected history metadata rows, got: %s", historyOutput)
-	}
-
-	countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s FOR VERSION AS OF %d", schemaName, trinoCustomersTable, snapshotID))
-	versionCount := mustParseCSVInt64(t, countOutput)
-	if versionCount != 8 {
-		t.Fatalf("expected 8 rows for version time travel, got %d", versionCount)
-	}
-
-	// Use current_timestamp - interval '1 second' to ensure it's in the past (Iceberg requirement)
-	countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s FOR TIMESTAMP AS OF (current_timestamp - interval '1' second)", schemaName, trinoCustomersTable))
-	timestampCount := mustParseCSVInt64(t, countOutput)
-	if timestampCount != 8 {
-		t.Fatalf("expected 8 rows for timestamp time travel, got %d", timestampCount)
-	}
-
-	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DELETE FROM iceberg.%s.%s WHERE customer_sk = 8", schemaName, trinoCustomersTable))
-	countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable))
-	rowCount = mustParseCSVInt64(t, countOutput)
-	if rowCount != 7 {
-		t.Fatalf("expected 7 rows after delete, got %d", rowCount)
-	}
-
-	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("ALTER TABLE iceberg.%s.%s EXECUTE rollback_to_snapshot(%d)", schemaName, trinoCustomersTable, snapshotID))
-	countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable))
-	rowCount = mustParseCSVInt64(t, countOutput)
-	if rowCount != 8 {
-		t.Fatalf("expected 8 rows after rollback, got %d", rowCount)
-	}
-}
-
-func runTrinoSQLAllowExists(t *testing.T, containerName, sql string) string {
-	t.Helper()
-
-	cmd := exec.Command("docker", "exec", containerName,
-		"trino", "--catalog", "iceberg",
-		"--output-format", "CSV",
-		"--execute", sql,
-	)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		outputStr := string(output)
-		if strings.Contains(outputStr, "already exists") {
-			return sanitizeTrinoOutput(outputStr)
-		}
-		t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, outputStr)
-	}
-	return sanitizeTrinoOutput(string(output))
-}
-
-func runTrinoSQLAllowNamespaceNotEmpty(t *testing.T, containerName, sql string) string {
-	t.Helper()
-
-	var output []byte
-	var err error
-	for attempt := 0; attempt < 3; attempt++ {
-		cmd := exec.Command("docker", "exec", containerName,
-			"trino", "--catalog", "iceberg",
-			"--output-format", "CSV",
-			"--execute", sql,
-		)
-		output, err = cmd.CombinedOutput()
-		if err == nil {
-			return sanitizeTrinoOutput(string(output))
-		}
-		outputStr := string(output)
-		if !strings.Contains(outputStr, "Namespace is not empty") {
-			t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, outputStr)
-		}
-		time.Sleep(500 * time.Millisecond)
-	}
-	t.Logf("Ignoring cleanup error for SQL %s: %s", sql, sanitizeTrinoOutput(string(output)))
-	return sanitizeTrinoOutput(string(output))
-}
-
-func runTrinoCTAS(t *testing.T, containerName, createSQL, deleteSQL, insertSQL string) {
-	t.Helper()
-
-	cmd := exec.Command("docker", "exec", containerName,
-		"trino", "--catalog", "iceberg",
-		"--output-format", "CSV",
-		"--execute", createSQL,
-	)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		outputStr := string(output)
-		if strings.Contains(outputStr, "already exists") {
-			if deleteSQL != "" {
-				runTrinoSQL(t, containerName, deleteSQL)
-			}
-			runTrinoSQL(t, containerName, insertSQL)
-			return
-		}
-		t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, createSQL, outputStr)
-	}
-}
-
-func hasCSVDataRow(output string) bool {
-	lines := strings.Split(strings.TrimSpace(output), "\n")
-	if len(lines) == 0 {
-		return false
-	}
-	for _, line := range lines {
-		if strings.TrimSpace(line) != "" {
-			return true
-		}
-	}
-	return false
-}
-
-func mustParseCSVInt64(t *testing.T, output string) int64 {
-	t.Helper()
-	value := mustFirstCSVValue(t, output)
-	parsed, err := strconv.ParseInt(value, 10, 64)
-	if err != nil {
-		t.Fatalf("failed to parse int from output %q: %v", output, err)
-	}
-	return parsed
-}
-
-func mustFirstCSVValue(t *testing.T, output string) string {
-	t.Helper()
-	lines := strings.Split(strings.TrimSpace(output), "\n")
-	if len(lines) == 0 {
-		t.Fatalf("expected CSV output with data row, got: %q", output)
-	}
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		parts := strings.Split(line, ",")
-		return strings.Trim(parts[0], "\"")
-	}
-	t.Fatalf("no CSV data rows found in output: %q", output)
-	return ""
-}

test/s3tables/catalog_trino/trino_catalog_test.go

@@ -17,7 +17,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
 )
 
 type TestEnvironment struct {
@@ -58,10 +57,12 @@ func TestTrinoIcebergCatalog(t *testing.T) {
 	env.StartSeaweedFS(t)
 	fmt.Printf(">>> SeaweedFS started.\n")
 
+	catalogBucket := "warehouse"
 	tableBucket := "iceberg-tables"
-	catalogBucket := tableBucket
 	fmt.Printf(">>> Creating table bucket: %s\n", tableBucket)
 	createTableBucket(t, env, tableBucket)
+	fmt.Printf(">>> Creating table bucket: %s\n", catalogBucket)
+	createTableBucket(t, env, catalogBucket)
 	fmt.Printf(">>> All buckets created.\n")
 
 	// Test Iceberg REST API directly
@@ -116,7 +117,7 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 		t.Fatalf("Failed to create temp dir: %v", err)
 	}
 
-	bindIP := testutil.FindBindIP()
+	bindIP := findBindIP()
 
 	masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
 	volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
@@ -148,8 +149,29 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	t.Helper()
 
 	// Create IAM config file
-	iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
-	if err != nil {
+	iamConfigPath := filepath.Join(env.dataDir, "iam_config.json")
+	iamConfig := fmt.Sprintf(`{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "%s",
+          "secretKey": "%s"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ]
+}`, env.accessKey, env.secretKey)
+
+	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
 		t.Fatalf("Failed to create IAM config: %v", err)
 	}
 
@@ -184,8 +206,6 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	cmd.Env = append(os.Environ(),
 		"AWS_ACCESS_KEY_ID="+env.accessKey,
 		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
-		"ICEBERG_WAREHOUSE=s3://iceberg-tables",
-		"S3TABLES_DEFAULT_BUCKET=iceberg-tables",
 	)
 
 	if err := cmd.Start(); err != nil {
@@ -262,13 +282,12 @@ func testIcebergRestAPI(t *testing.T, env *TestEnvironment) {
 	fmt.Printf(">>> Testing Iceberg REST API directly...\n")
 
 	// First, verify the service is listening
-	addr := net.JoinHostPort(env.bindIP, fmt.Sprintf("%d", env.icebergPort))
-	conn, err := net.Dial("tcp", addr)
+	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", env.bindIP, env.icebergPort))
 	if err != nil {
-		t.Fatalf("Cannot connect to Iceberg service at %s: %v", addr, err)
+		t.Fatalf("Cannot connect to Iceberg service at %s:%d: %v", env.bindIP, env.icebergPort, err)
 	}
 	conn.Close()
-	t.Logf("Successfully connected to Iceberg service at %s", addr)
+	t.Logf("Successfully connected to Iceberg service at %s:%d", env.bindIP, env.icebergPort)
 
 	// Test /v1/config endpoint
 	url := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
@@ -300,9 +319,8 @@ func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket strin
 	config := fmt.Sprintf(`connector.name=iceberg
 iceberg.catalog.type=rest
 iceberg.rest-catalog.uri=http://host.docker.internal:%d
-iceberg.rest-catalog.warehouse=s3://%s
+iceberg.rest-catalog.warehouse=s3://%s/
 iceberg.file-format=PARQUET
-iceberg.unique-table-location=true
 
 # S3 storage config
 fs.native-s3.enabled=true
@@ -380,7 +398,8 @@ func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
 		return
 	}
 
-	t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s", string(lastOutput))
+	logs, _ := exec.Command("docker", "logs", containerName).CombinedOutput()
+	t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s\nTrino logs:\n%s", string(lastOutput), string(logs))
 }
 
 func runTrinoSQL(t *testing.T, containerName, sql string) string {
@@ -393,9 +412,10 @@ func runTrinoSQL(t *testing.T, containerName, sql string) string {
 	)
 	output, err := cmd.CombinedOutput()
 	if err != nil {
-		t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, string(output))
+		logs, _ := exec.Command("docker", "logs", containerName).CombinedOutput()
+		t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s\nTrino logs:\n%s", err, sql, string(output), string(logs))
 	}
-	return sanitizeTrinoOutput(string(output))
+	return string(output)
 }
 
 func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
@@ -419,30 +439,6 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	t.Logf("Created table bucket: %s", bucketName)
 }
 
-func sanitizeTrinoOutput(output string) string {
-	lines := strings.Split(strings.TrimSpace(output), "\n")
-	filtered := make([]string, 0, len(lines))
-	for _, line := range lines {
-		if strings.Contains(line, "org.jline.utils.Log") {
-			continue
-		}
-		if strings.Contains(line, "Unable to create a system terminal") {
-			continue
-		}
-		if strings.HasPrefix(line, "WARNING:") {
-			continue
-		}
-		if strings.TrimSpace(line) == "" {
-			continue
-		}
-		filtered = append(filtered, line)
-	}
-	if len(filtered) == 0 {
-		return ""
-	}
-	return strings.Join(filtered, "\n") + "\n"
-}
-
 func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	t.Helper()
 
@@ -514,6 +510,25 @@ func getFreePort() (int, error) {
 	return addr.Port, nil
 }
 
+func findBindIP() string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "127.0.0.1"
+	}
+	for _, addr := range addrs {
+		ipNet, ok := addr.(*net.IPNet)
+		if !ok || ipNet.IP == nil {
+			continue
+		}
+		ip := ipNet.IP.To4()
+		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
+			continue
+		}
+		return ip.String()
+	}
+	return "127.0.0.1"
+}
+
 func randomString(length int) string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
 	b := make([]byte, length)

test/s3tables/catalog_trino/trino_crud_operations_test.go

@@ -24,9 +24,10 @@ func setupTrinoTest(t *testing.T) *TestEnvironment {
 	t.Logf(">>> Starting SeaweedFS...")
 	env.StartSeaweedFS(t)
 
+	catalogBucket := "warehouse"
 	tableBucket := "iceberg-tables"
-	catalogBucket := tableBucket
 	createTableBucket(t, env, tableBucket)
+	createTableBucket(t, env, catalogBucket)
 
 	configDir := env.writeTrinoConfig(t, catalogBucket)
 	env.startTrinoContainer(t, configDir)

test/s3tables/lakekeeper/lakekeeper_test.go

@@ -1,341 +0,0 @@
-package lakekeeper
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
-)
-
-type TestEnvironment struct {
-	seaweedDir     string
-	weedBinary     string
-	dataDir        string
-	bindIP         string
-	s3Port         int
-	s3GrpcPort     int
-	masterPort     int
-	masterGrpcPort int
-	filerPort      int
-	filerGrpcPort  int
-	volumePort     int
-	volumeGrpcPort int
-	weedProcess    *exec.Cmd
-	weedCancel     context.CancelFunc
-	accessKey      string
-	secretKey      string
-}
-
-func TestLakekeeperIntegration(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	if !testutil.HasDocker() {
-		t.Skip("Docker not available, skipping Lakekeeper integration test")
-	}
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
-	env.StartSeaweedFS(t)
-	fmt.Printf(">>> SeaweedFS started.\n")
-
-	// Run python script in docker to test STS and S3 operations
-	runLakekeeperRepro(t, env)
-}
-
-func NewTestEnvironment(t *testing.T) *TestEnvironment {
-	t.Helper()
-
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("Failed to get working directory: %v", err)
-	}
-
-	seaweedDir := wd
-	for i := 0; i < 6; i++ {
-		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
-			break
-		}
-		seaweedDir = filepath.Dir(seaweedDir)
-	}
-
-	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
-	if _, err := os.Stat(weedBinary); err != nil {
-		weedBinary = "weed"
-		if _, err := exec.LookPath(weedBinary); err != nil {
-			t.Skip("weed binary not found, skipping integration test")
-		}
-	}
-
-	dataDir, err := os.MkdirTemp("", "seaweed-lakekeeper-test-*")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %v", err)
-	}
-
-	bindIP := testutil.FindBindIP()
-
-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
-
-	return &TestEnvironment{
-		seaweedDir:     seaweedDir,
-		weedBinary:     weedBinary,
-		dataDir:        dataDir,
-		bindIP:         bindIP,
-		s3Port:         s3Port,
-		s3GrpcPort:     s3GrpcPort,
-		masterPort:     masterPort,
-		masterGrpcPort: masterGrpcPort,
-		filerPort:      filerPort,
-		filerGrpcPort:  filerGrpcPort,
-		volumePort:     volumePort,
-		volumeGrpcPort: volumeGrpcPort,
-		accessKey:      "admin",
-		secretKey:      "admin",
-	}
-}
-
-func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
-	t.Helper()
-
-	iamConfigPath := filepath.Join(env.dataDir, "iam.json")
-	// Note: signingKey must be base64 encoded for []byte JSON unmarshaling
-	iamConfig := fmt.Sprintf(`{
-  "identities": [
-    {
-      "name": "admin",
-      "credentials": [
-        {
-          "accessKey": "%s",
-          "secretKey": "%s"
-        }
-      ],
-      "actions": ["Admin", "Read", "List", "Tagging", "Write"]
-    }
-  ],
-  "sts": {
-    "tokenDuration": "12h",
-    "maxSessionLength": "24h",
-    "issuer": "seaweedfs-sts",
-    "signingKey": "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz"
-  },
-  "roles": [
-    {
-      "roleName": "LakekeeperVendedRole",
-      "roleArn": "arn:aws:iam::000000000000:role/LakekeeperVendedRole",
-      "trustPolicy": {
-        "Version": "2012-10-17",
-        "Statement": [
-          {
-            "Effect": "Allow",
-            "Principal": "*",
-            "Action": "sts:AssumeRole"
-          }
-        ]
-      },
-      "attachedPolicies": ["FullAccess"]
-    }
-  ],
-  "policies": [
-    {
-      "name": "FullAccess",
-      "document": {
-        "Version": "2012-10-17",
-        "Statement": [
-          {
-            "Effect": "Allow",
-            "Action": "*",
-            "Resource": "*"
-          }
-        ]
-      }
-    }
-  ]
-}`, env.accessKey, env.secretKey)
-
-	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
-		t.Fatalf("Failed to create IAM config: %v", err)
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	env.weedCancel = cancel
-
-	// Start weed mini with both S3 config (standard IAM) and IAM config (advanced IAM/STS)
-	cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
-		"-master.port", fmt.Sprintf("%d", env.masterPort),
-		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
-		"-volume.port", fmt.Sprintf("%d", env.volumePort),
-		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
-		"-filer.port", fmt.Sprintf("%d", env.filerPort),
-		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
-		"-s3.port", fmt.Sprintf("%d", env.s3Port),
-		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
-		"-s3.config", iamConfigPath,
-		"-s3.iam.config", iamConfigPath,
-		"-s3.iam.readOnly=false",
-		"-ip", env.bindIP,
-		"-ip.bind", "0.0.0.0",
-		"-dir", env.dataDir,
-	)
-	cmd.Dir = env.dataDir
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-
-	if err := cmd.Start(); err != nil {
-		t.Fatalf("Failed to start SeaweedFS: %v", err)
-	}
-	env.weedProcess = cmd
-
-	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
-		t.Fatalf("S3 API failed to become ready")
-	}
-}
-
-func (env *TestEnvironment) Cleanup(t *testing.T) {
-	t.Helper()
-	if env.weedCancel != nil {
-		env.weedCancel()
-	}
-	if env.weedProcess != nil {
-		time.Sleep(1 * time.Second)
-		_ = env.weedProcess.Wait()
-	}
-	if env.dataDir != "" {
-		_ = os.RemoveAll(env.dataDir)
-	}
-}
-
-func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
-	t.Helper()
-
-	scriptContent := fmt.Sprintf(`
-import boto3
-import botocore.config
-import botocore
-from botocore.exceptions import ClientError
-import os
-import sys
-import time
-import logging
-
-# Enable botocore debug logging to see signature calculation
-logging.basicConfig(level=logging.DEBUG)
-botocore.session.get_session().set_debug_logger()
-
-print("Starting Lakekeeper repro test...")
-
-endpoint_url = "http://host.docker.internal:%d"
-access_key = "%s"
-secret_key = "%s"
-region = "us-east-1"
-
-print(f"Connecting to {endpoint_url}")
-
-try:
-    config = botocore.config.Config(
-        retries={'max_attempts': 3}
-    )
-    sts = boto3.client(
-        'sts',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-        config=config
-    )
-
-    role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
-    session_name = "lakekeeper-session"
-
-    print(f"Calling AssumeRole on {role_arn} with POST body...")
-    
-    # Standard boto3 call sends parameters in POST body
-    response = sts.assume_role(
-        RoleArn=role_arn,
-        RoleSessionName=session_name
-    )
-
-    creds = response['Credentials']
-    access_key_id = creds['AccessKeyId']
-    secret_access_key = creds['SecretAccessKey']
-    session_token = creds['SessionToken']
-
-    print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
-    
-    if not access_key_id.startswith("ASIA"):
-        print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
-        sys.exit(1)
-
-    print("Verifying S3 operations with vended credentials...")
-    s3 = boto3.client(
-        's3',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=access_key_id,
-        aws_secret_access_key=secret_access_key,
-        aws_session_token=session_token,
-        region_name=region,
-        config=config
-    )
-
-    bucket = "lakekeeper-vended-bucket"
-    print(f"Creating bucket {bucket}...")
-    s3.create_bucket(Bucket=bucket)
-
-    print("Listing buckets...")
-    response = s3.list_buckets()
-    buckets = [b['Name'] for b in response['Buckets']]
-    print(f"Found buckets: {buckets}")
-
-    if bucket not in buckets:
-        print(f"FAILED: Bucket {bucket} not found in list")
-        sys.exit(1)
-
-    print("SUCCESS: Lakekeeper flow verified!")
-    sys.exit(0)
-    
-except Exception as e:
-    print(f"FAILED: {e}")
-    # Print more details if it is a ClientError
-    if hasattr(e, 'response'):
-        print(f"Response: {e.response}")
-    sys.exit(1)
-`, env.s3Port, env.accessKey, env.secretKey)
-
-	scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
-	if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
-		t.Fatalf("Failed to write python script: %v", err)
-	}
-
-	containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
-
-	// Create a context with timeout for the docker run command
-	dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	defer dockerCancel()
-
-	cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
-		"--name", containerName,
-		"--add-host", "host.docker.internal:host-gateway",
-		"-v", fmt.Sprintf("%s:/work", env.dataDir),
-		"python:3",
-		"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
-	)
-
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		if dockerCtx.Err() == context.DeadlineExceeded {
-			t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
-		}
-		t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
-	}
-	t.Logf("Lakekeeper repro client output:\n%s", string(output))
-}

test/s3tables/sts_integration/sts_integration_test.go

@@ -1,409 +0,0 @@
-package sts_integration
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
-)
-
-// TestEnvironment mirrors the one in trino_catalog_test.go but simplified
-type TestEnvironment struct {
-	seaweedDir      string
-	weedBinary      string
-	dataDir         string
-	bindIP          string
-	s3Port          int
-	s3GrpcPort      int
-	masterPort      int
-	masterGrpcPort  int
-	filerPort       int
-	filerGrpcPort   int
-	volumePort      int
-	volumeGrpcPort  int
-	weedProcess     *exec.Cmd
-	weedCancel      context.CancelFunc
-	dockerAvailable bool
-	accessKey       string
-	secretKey       string
-}
-
-const testSTSIntegrationSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test-signing-key-for-sts-integration-tests
-
-func TestSTSIntegration(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env := NewTestEnvironment(t)
-	defer env.Cleanup(t)
-
-	if !env.dockerAvailable {
-		t.Skip("Docker not available, skipping STS integration test")
-	}
-
-	fmt.Printf(">>> Starting SeaweedFS...\n")
-	env.StartSeaweedFS(t)
-	fmt.Printf(">>> SeaweedFS started.\n")
-
-	// Run python script in docker to test STS
-	runPythonSTSClient(t, env)
-}
-
-func NewTestEnvironment(t *testing.T) *TestEnvironment {
-	t.Helper()
-
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("Failed to get working directory: %v", err)
-	}
-
-	seaweedDir := wd
-	for i := 0; i < 6; i++ {
-		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
-			break
-		}
-		seaweedDir = filepath.Dir(seaweedDir)
-	}
-
-	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
-	info, err := os.Stat(weedBinary)
-	if err != nil || info.IsDir() {
-		weedBinary = "weed"
-		if _, err := exec.LookPath(weedBinary); err != nil {
-			t.Skip("weed binary not found, skipping integration test")
-		}
-	}
-
-	if !testutil.HasDocker() {
-		t.Skip("Docker not available, skipping integration test")
-	}
-
-	// Create a unique temporary directory for this test run
-	dataDir, err := os.MkdirTemp("", "seaweed-sts-test-*")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %v", err)
-	}
-	// The Cleanup method will remove this directory, so no need for defer here.
-
-	bindIP := testutil.FindBindIP()
-
-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair
-
-	return &TestEnvironment{
-		seaweedDir:      seaweedDir,
-		weedBinary:      weedBinary,
-		dataDir:         dataDir,
-		bindIP:          bindIP,
-		s3Port:          s3Port,
-		s3GrpcPort:      s3GrpcPort,
-		masterPort:      masterPort,
-		masterGrpcPort:  masterGrpcPort,
-		filerPort:       filerPort,
-		filerGrpcPort:   filerGrpcPort,
-		volumePort:      volumePort,
-		volumeGrpcPort:  volumeGrpcPort,
-		dockerAvailable: testutil.HasDocker(),
-		accessKey:       "admin",
-		secretKey:       "adminadmin",
-	}
-}
-
-func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
-	t.Helper()
-
-	iamConfigPath := filepath.Join(env.dataDir, "iam.json")
-	// Note: signingKey must be base64 encoded for []byte JSON unmarshaling
-	iamConfig := fmt.Sprintf(`{
-  "identities": [
-    {
-      "name": "admin",
-      "credentials": [
-        { "accessKey": "%s", "secretKey": "%s" }
-      ],
-      "actions": ["Admin", "Read", "Write", "List", "Tagging"]
-    }
-  ],
-  "sts": {
-    "tokenDuration": "1h",
-    "maxSessionLength": "12h",
-    "issuer": "seaweedfs-sts",
-    "signingKey": "%s"
-  },
-  "policy": {
-    "defaultEffect": "Deny",
-    "storeType": "memory"
-  },
-  "policies": [
-    {
-      "name": "S3FullAccessPolicy",
-      "document": {
-        "Version": "2012-10-17",
-        "Statement": [
-          {
-            "Effect": "Allow",
-            "Action": ["s3:*"],
-            "Resource": ["*"]
-          }
-        ]
-      }
-    }
-  ],
-  "roles": [
-    {
-      "roleName": "TestRole",
-      "roleArn": "arn:aws:iam::role/TestRole",
-      "attachedPolicies": ["S3FullAccessPolicy"],
-      "trustPolicy": {
-        "Version": "2012-10-17",
-        "Statement": [
-          {
-            "Effect": "Allow",
-            "Principal": "*",
-            "Action": ["sts:AssumeRole"]
-          }
-        ]
-      }
-    }
-  ]
-}`, env.accessKey, env.secretKey, testSTSIntegrationSigningKey)
-	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
-		t.Fatalf("Failed to create IAM config: %v", err)
-	}
-
-	// Create empty security.toml
-	securityToml := filepath.Join(env.dataDir, "security.toml")
-	if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
-		t.Fatalf("Failed to create security.toml: %v", err)
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	env.weedCancel = cancel
-
-	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
-		"-master.port", fmt.Sprintf("%d", env.masterPort),
-		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
-		"-volume.port", fmt.Sprintf("%d", env.volumePort),
-		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
-		"-filer.port", fmt.Sprintf("%d", env.filerPort),
-		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
-		"-s3.port", fmt.Sprintf("%d", env.s3Port),
-		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
-		"-s3.config", iamConfigPath,
-		"-s3.iam.config", iamConfigPath,
-		"-s3.iam.readOnly", "false",
-		"-ip", env.bindIP,
-		"-ip.bind", "0.0.0.0",
-		"-dir", env.dataDir,
-	)
-	cmd.Dir = env.dataDir
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-
-	if err := cmd.Start(); err != nil {
-		t.Fatalf("Failed to start SeaweedFS: %v", err)
-	}
-	env.weedProcess = cmd
-
-	// Wait for S3 API to be ready
-	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
-		t.Fatalf("S3 API failed to become ready")
-	}
-}
-
-func (env *TestEnvironment) Start(t *testing.T) {
-	if !testutil.HasDocker() {
-		t.Skip("Docker not available")
-	}
-}
-
-func (env *TestEnvironment) Cleanup(t *testing.T) {
-	t.Helper()
-	if env.weedCancel != nil {
-		env.weedCancel()
-	}
-	if env.weedProcess != nil {
-		time.Sleep(1 * time.Second)
-		_ = env.weedProcess.Wait()
-	}
-	if env.dataDir != "" {
-		_ = os.RemoveAll(env.dataDir)
-	}
-}
-
-func runPythonSTSClient(t *testing.T, env *TestEnvironment) {
-	t.Helper()
-
-	// Write python script to temp dir
-	scriptContent := fmt.Sprintf(`
-import boto3
-import botocore.config
-from botocore.exceptions import ClientError
-import json
-import sys
-import time
-import urllib.error
-import urllib.request
-
-print("Starting STS inline session policy test...")
-
-primary_endpoint = "http://host.docker.internal:%d"
-fallback_endpoint = "http://%s:%d"
-access_key = "%s"
-secret_key = "%s"
-region = "us-east-1"
-
-try:
-    def wait_for_endpoint(url, timeout=30):
-        deadline = time.time() + timeout
-        while time.time() < deadline:
-            try:
-                with urllib.request.urlopen(url, timeout=2):
-                    return True
-            except urllib.error.HTTPError:
-                return True
-            except Exception:
-                time.sleep(1)
-        return False
-
-    def select_endpoint(urls):
-        for url in urls:
-            if wait_for_endpoint(url):
-                return url
-        raise Exception("No reachable S3 endpoint from container")
-
-    endpoint_url = select_endpoint([primary_endpoint, fallback_endpoint])
-    print(f"Using endpoint {endpoint_url}")
-
-    config = botocore.config.Config(
-        retries={'max_attempts': 0},
-        s3={'addressing_style': 'path'}
-    )
-    admin_s3 = boto3.client(
-        's3',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-        config=config
-    )
-
-    bucket = f"sts-inline-policy-{int(time.time() * 1000)}"
-    key = "allowed.txt"
-
-    print(f"Creating bucket {bucket} with admin credentials")
-    admin_s3.create_bucket(Bucket=bucket)
-    admin_s3.put_object(Bucket=bucket, Key=key, Body=b"ok")
-
-    sts = boto3.client(
-        'sts',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-        config=config
-    )
-
-    role_arn = "arn:aws:iam::role/TestRole"
-    session_name = "test-session"
-    session_policy = json.dumps({
-        "Version": "2012-10-17",
-        "Statement": [
-            {
-                "Effect": "Allow",
-                "Action": ["s3:ListBucket"],
-                "Resource": [f"arn:aws:s3:::{bucket}"]
-            },
-            {
-                "Effect": "Allow",
-                "Action": ["s3:GetObject"],
-                "Resource": [f"arn:aws:s3:::{bucket}/*"]
-            }
-        ]
-    })
-
-    print(f"Calling AssumeRole on {role_arn} with inline session policy")
-    response = sts.assume_role(
-        RoleArn=role_arn,
-        RoleSessionName=session_name,
-        Policy=session_policy
-    )
-
-    creds = response['Credentials']
-    vended_s3 = boto3.client(
-        's3',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=creds['AccessKeyId'],
-        aws_secret_access_key=creds['SecretAccessKey'],
-        aws_session_token=creds['SessionToken'],
-        region_name=region,
-        config=config
-    )
-
-    print("Listing objects (allowed)")
-    list_resp = vended_s3.list_objects_v2(Bucket=bucket)
-    keys = [obj.get('Key') for obj in list_resp.get('Contents', [])]
-    if key not in keys:
-        print(f"FAILED: Expected to see {key} in list_objects_v2 results")
-        sys.exit(1)
-
-    print("Getting object (allowed)")
-    body = vended_s3.get_object(Bucket=bucket, Key=key)['Body'].read()
-    if body != b"ok":
-        print("FAILED: Unexpected object content")
-        sys.exit(1)
-
-    print("Putting object (expected to be denied)")
-    try:
-        vended_s3.put_object(Bucket=bucket, Key="denied.txt", Body=b"no")
-        print("FAILED: PutObject unexpectedly succeeded")
-        sys.exit(1)
-    except ClientError as e:
-        error_code = e.response.get('Error', {}).get('Code', '')
-        if error_code != 'AccessDenied':
-            print(f"FAILED: Expected AccessDenied, got {error_code}")
-            sys.exit(1)
-        print("PutObject correctly denied by inline session policy")
-
-    print("SUCCESS: Inline session policy downscoping verified")
-    sys.exit(0)
-except Exception as e:
-    print(f"FAILED: {e}")
-    if hasattr(e, 'response'):
-        print(f"Response: {e.response}")
-    import traceback
-    traceback.print_exc()
-    sys.exit(1)
-`, env.s3Port, env.bindIP, env.s3Port, env.accessKey, env.secretKey)
-
-	scriptPath := filepath.Join(env.dataDir, "sts_test.py")
-	if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
-		t.Fatalf("Failed to write python script: %v", err)
-	}
-
-	containerName := "seaweed-sts-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
-
-	cmd := exec.Command("docker", "run", "--rm",
-		"--name", containerName,
-		"--add-host", "host.docker.internal:host-gateway",
-		"-v", fmt.Sprintf("%s:/work", env.dataDir),
-		"python:3",
-		"/bin/bash", "-c", "pip install boto3 && python /work/sts_test.py",
-	)
-
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("Python STS client failed: %v\nOutput:\n%s", err, string(output))
-	}
-	t.Logf("Python STS client output:\n%s", string(output))
-}
-
-// Helpers copied from trino_catalog_test.go

test/s3tables/table-buckets/s3tables_integration_test.go

@@ -556,14 +556,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
 	}
 
-	// Set environment variables for admin credentials safely for this test
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
-		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
-	}
-	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
-	}
-
 	// Start weed mini in a goroutine by calling the command directly
 	cluster.wg.Add(1)
 	go func() {

test/s3tables/testutil/docker.go

@@ -1,66 +0,0 @@
-package testutil
-
-import (
-	"context"
-	"net"
-	"net/http"
-	"os/exec"
-	"testing"
-	"time"
-)
-
-func HasDocker() bool {
-	cmd := exec.Command("docker", "version")
-	return cmd.Run() == nil
-}
-
-func MustFreePortPair(t *testing.T, name string) (int, int) {
-	httpPort, grpcPort, err := findAvailablePortPair()
-	if err != nil {
-		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
-	}
-	return httpPort, grpcPort
-}
-
-func findAvailablePortPair() (int, int, error) {
-	httpPort, err := GetFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	grpcPort, err := GetFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	return httpPort, grpcPort, nil
-}
-
-func GetFreePort() (int, error) {
-	listener, err := net.Listen("tcp", "0.0.0.0:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
-	return listener.Addr().(*net.TCPAddr).Port, nil
-}
-
-func WaitForService(url string, timeout time.Duration) bool {
-	client := &http.Client{Timeout: 2 * time.Second}
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return false
-		case <-ticker.C:
-			resp, err := client.Get(url)
-			if err == nil {
-				resp.Body.Close()
-				return true
-			}
-		}
-	}
-}

test/s3tables/testutil/weed_mini.go

@@ -1,56 +0,0 @@
-package testutil
-
-import (
-	"fmt"
-	"net"
-	"os"
-	"path/filepath"
-)
-
-func FindBindIP() string {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return "127.0.0.1"
-	}
-	for _, addr := range addrs {
-		ipNet, ok := addr.(*net.IPNet)
-		if !ok || ipNet.IP == nil {
-			continue
-		}
-		ip := ipNet.IP.To4()
-		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
-			continue
-		}
-		return ip.String()
-	}
-	return "127.0.0.1"
-}
-
-func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
-	iamConfigPath := filepath.Join(dir, "iam_config.json")
-	iamConfig := fmt.Sprintf(`{
-  "identities": [
-    {
-      "name": "admin",
-      "credentials": [
-        {
-          "accessKey": "%s",
-          "secretKey": "%s"
-        }
-      ],
-      "actions": [
-        "Admin",
-        "Read",
-        "List",
-        "Tagging",
-        "Write"
-      ]
-    }
-  ]
-}`, accessKey, secretKey)
-
-	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
-		return "", err
-	}
-	return iamConfigPath, nil
-}

test/volume_server/Makefile

@@ -1,7 +0,0 @@
-.PHONY: test-volume-server test-volume-server-short
-
-test-volume-server:
-	go test ./test/volume_server/... -v
-
-test-volume-server-short:
-	go test ./test/volume_server/... -short -v

test/volume_server/README.md

@@ -1,27 +0,0 @@
-# Volume Server Integration Tests
-
-This package contains integration tests for SeaweedFS volume server HTTP and gRPC APIs.
-
-## Run Tests
-
-Run tests from repo root:
-
-```bash
-go test ./test/volume_server/... -v
-```
-
-If a `weed` binary is not found, the harness will build one automatically.
-
-## Optional environment variables
-
-- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build).
-- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs.
-
-## Current scope (Phase 0)
-
-- Shared cluster/framework utilities
-- Matrix profile definitions
-- Initial HTTP admin endpoint checks
-- Initial gRPC state/status checks
-
-More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`.

test/volume_server/framework/cluster.go

@@ -1,442 +0,0 @@
-package framework
-
-import (
-	"bufio"
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-const (
-	defaultWaitTimeout    = 30 * time.Second
-	defaultWaitTick       = 200 * time.Millisecond
-	testVolumeSizeLimitMB = 32
-)
-
-// Cluster is a lightweight SeaweedFS master + one volume server test harness.
-type Cluster struct {
-	testingTB testing.TB
-	profile   matrix.Profile
-
-	weedBinary string
-	baseDir    string
-	configDir  string
-	logsDir    string
-	keepLogs   bool
-
-	masterPort     int
-	masterGrpcPort int
-	volumePort     int
-	volumeGrpcPort int
-	volumePubPort  int
-
-	masterCmd *exec.Cmd
-	volumeCmd *exec.Cmd
-
-	cleanupOnce sync.Once
-}
-
-// StartSingleVolumeCluster boots one master and one volume server.
-func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
-	t.Helper()
-
-	weedBinary, err := FindOrBuildWeedBinary()
-	if err != nil {
-		t.Fatalf("resolve weed binary: %v", err)
-	}
-
-	baseDir, keepLogs, err := newWorkDir()
-	if err != nil {
-		t.Fatalf("create temp test directory: %v", err)
-	}
-
-	configDir := filepath.Join(baseDir, "config")
-	logsDir := filepath.Join(baseDir, "logs")
-	masterDataDir := filepath.Join(baseDir, "master")
-	volumeDataDir := filepath.Join(baseDir, "volume")
-	for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} {
-		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
-			t.Fatalf("create %s: %v", dir, mkErr)
-		}
-	}
-
-	if err = writeSecurityConfig(configDir, profile); err != nil {
-		t.Fatalf("write security config: %v", err)
-	}
-
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
-	if err != nil {
-		t.Fatalf("allocate master port pair: %v", err)
-	}
-
-	ports, err := allocatePorts(3)
-	if err != nil {
-		t.Fatalf("allocate ports: %v", err)
-	}
-
-	c := &Cluster{
-		testingTB:      t,
-		profile:        profile,
-		weedBinary:     weedBinary,
-		baseDir:        baseDir,
-		configDir:      configDir,
-		logsDir:        logsDir,
-		keepLogs:       keepLogs,
-		masterPort:     masterPort,
-		masterGrpcPort: masterGrpcPort,
-		volumePort:     ports[0],
-		volumeGrpcPort: ports[1],
-		volumePubPort:  ports[0],
-	}
-	if profile.SplitPublicPort {
-		c.volumePubPort = ports[2]
-	}
-
-	if err = c.startMaster(masterDataDir); err != nil {
-		c.Stop()
-		t.Fatalf("start master: %v", err)
-	}
-	if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
-		masterLog := c.tailLog("master.log")
-		c.Stop()
-		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
-	}
-
-	if err = c.startVolume(volumeDataDir); err != nil {
-		masterLog := c.tailLog("master.log")
-		c.Stop()
-		t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog)
-	}
-	if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
-		volumeLog := c.tailLog("volume.log")
-		c.Stop()
-		t.Fatalf("wait for volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
-	}
-	if err = c.waitForTCP(c.VolumeGRPCAddress()); err != nil {
-		volumeLog := c.tailLog("volume.log")
-		c.Stop()
-		t.Fatalf("wait for volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
-	}
-
-	t.Cleanup(func() {
-		c.Stop()
-	})
-
-	return c
-}
-
-// Stop terminates all processes and cleans temporary files.
-func (c *Cluster) Stop() {
-	if c == nil {
-		return
-	}
-	c.cleanupOnce.Do(func() {
-		stopProcess(c.volumeCmd)
-		stopProcess(c.masterCmd)
-		if !c.keepLogs && !c.testingTB.Failed() {
-			_ = os.RemoveAll(c.baseDir)
-		} else if c.baseDir != "" {
-			c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
-		}
-	})
-}
-
-func (c *Cluster) startMaster(dataDir string) error {
-	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
-	if err != nil {
-		return err
-	}
-
-	args := []string{
-		"-config_dir=" + c.configDir,
-		"master",
-		"-ip=127.0.0.1",
-		"-port=" + strconv.Itoa(c.masterPort),
-		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
-		"-mdir=" + dataDir,
-		"-peers=none",
-		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
-		"-defaultReplication=000",
-	}
-
-	c.masterCmd = exec.Command(c.weedBinary, args...)
-	c.masterCmd.Dir = c.baseDir
-	c.masterCmd.Stdout = logFile
-	c.masterCmd.Stderr = logFile
-	return c.masterCmd.Start()
-}
-
-func (c *Cluster) startVolume(dataDir string) error {
-	logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log"))
-	if err != nil {
-		return err
-	}
-
-	args := []string{
-		"-config_dir=" + c.configDir,
-		"volume",
-		"-ip=127.0.0.1",
-		"-port=" + strconv.Itoa(c.volumePort),
-		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort),
-		"-port.public=" + strconv.Itoa(c.volumePubPort),
-		"-dir=" + dataDir,
-		"-max=16",
-		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
-		"-readMode=" + c.profile.ReadMode,
-		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
-		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
-	}
-	if c.profile.InflightUploadTimeout > 0 {
-		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
-	}
-	if c.profile.InflightDownloadTimeout > 0 {
-		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
-	}
-
-	c.volumeCmd = exec.Command(c.weedBinary, args...)
-	c.volumeCmd.Dir = c.baseDir
-	c.volumeCmd.Stdout = logFile
-	c.volumeCmd.Stderr = logFile
-	return c.volumeCmd.Start()
-}
-
-func (c *Cluster) waitForHTTP(url string) error {
-	client := &http.Client{Timeout: 1 * time.Second}
-	deadline := time.Now().Add(defaultWaitTimeout)
-	for time.Now().Before(deadline) {
-		resp, err := client.Get(url)
-		if err == nil {
-			_, _ = io.Copy(io.Discard, resp.Body)
-			resp.Body.Close()
-			if resp.StatusCode < 500 {
-				return nil
-			}
-		}
-		time.Sleep(defaultWaitTick)
-	}
-	return fmt.Errorf("timed out waiting for %s", url)
-}
-
-func (c *Cluster) waitForTCP(addr string) error {
-	deadline := time.Now().Add(defaultWaitTimeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", addr, time.Second)
-		if err == nil {
-			_ = conn.Close()
-			return nil
-		}
-		time.Sleep(defaultWaitTick)
-	}
-	return fmt.Errorf("timed out waiting for tcp %s", addr)
-}
-
-func stopProcess(cmd *exec.Cmd) {
-	if cmd == nil || cmd.Process == nil {
-		return
-	}
-
-	_ = cmd.Process.Signal(os.Interrupt)
-	done := make(chan error, 1)
-	go func() {
-		done <- cmd.Wait()
-	}()
-
-	select {
-	case <-time.After(10 * time.Second):
-		_ = cmd.Process.Kill()
-		<-done
-	case <-done:
-	}
-}
-
-func allocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for _, ll := range listeners {
-				_ = ll.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
-	}
-	for _, l := range listeners {
-		_ = l.Close()
-	}
-	return ports, nil
-}
-
-func allocateMasterPortPair() (int, int, error) {
-	for masterPort := 10000; masterPort <= 55535; masterPort++ {
-		masterGrpcPort := masterPort + 10000
-		l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
-		if err != nil {
-			continue
-		}
-		l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort)))
-		if err != nil {
-			_ = l1.Close()
-			continue
-		}
-		_ = l2.Close()
-		_ = l1.Close()
-		return masterPort, masterGrpcPort, nil
-	}
-	return 0, 0, errors.New("unable to find available master port pair")
-}
-
-func newWorkDir() (dir string, keepLogs bool, err error) {
-	keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1"
-	dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_")
-	return dir, keepLogs, err
-}
-
-func writeSecurityConfig(configDir string, profile matrix.Profile) error {
-	var b strings.Builder
-	if profile.EnableJWT {
-		if profile.JWTSigningKey == "" || profile.JWTReadKey == "" {
-			return errors.New("jwt profile requires both write and read keys")
-		}
-		b.WriteString("[jwt.signing]\n")
-		b.WriteString("key = \"")
-		b.WriteString(profile.JWTSigningKey)
-		b.WriteString("\"\n")
-		b.WriteString("expires_after_seconds = 60\n\n")
-
-		b.WriteString("[jwt.signing.read]\n")
-		b.WriteString("key = \"")
-		b.WriteString(profile.JWTReadKey)
-		b.WriteString("\"\n")
-		b.WriteString("expires_after_seconds = 60\n")
-	}
-	if b.Len() == 0 {
-		b.WriteString("# optional security config generated for integration tests\n")
-	}
-	return os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(b.String()), 0o644)
-}
-
-// FindOrBuildWeedBinary returns an executable weed binary, building one when needed.
-func FindOrBuildWeedBinary() (string, error) {
-	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
-		if isExecutableFile(fromEnv) {
-			return fromEnv, nil
-		}
-		return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv)
-	}
-
-	repoRoot := ""
-	if _, file, _, ok := runtime.Caller(0); ok {
-		repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
-		candidate := filepath.Join(repoRoot, "weed", "weed")
-		if isExecutableFile(candidate) {
-			return candidate, nil
-		}
-	}
-
-	if repoRoot == "" {
-		return "", errors.New("unable to detect repository root")
-	}
-
-	binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin")
-	if err := os.MkdirAll(binDir, 0o755); err != nil {
-		return "", fmt.Errorf("create binary directory %s: %w", binDir, err)
-	}
-	binPath := filepath.Join(binDir, "weed")
-	if isExecutableFile(binPath) {
-		return binPath, nil
-	}
-
-	cmd := exec.Command("go", "build", "-o", binPath, ".")
-	cmd.Dir = filepath.Join(repoRoot, "weed")
-	var out bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	if err := cmd.Run(); err != nil {
-		return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String())
-	}
-	if !isExecutableFile(binPath) {
-		return "", fmt.Errorf("built weed binary is not executable: %s", binPath)
-	}
-	return binPath, nil
-}
-
-func isExecutableFile(path string) bool {
-	info, err := os.Stat(path)
-	if err != nil || info.IsDir() {
-		return false
-	}
-	mode := info.Mode().Perm()
-	return mode&0o111 != 0
-}
-
-func (c *Cluster) tailLog(logName string) string {
-	f, err := os.Open(filepath.Join(c.logsDir, logName))
-	if err != nil {
-		return ""
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-	lines := make([]string, 0, 40)
-	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
-		if len(lines) > 40 {
-			lines = lines[1:]
-		}
-	}
-	return strings.Join(lines, "\n")
-}
-
-func (c *Cluster) MasterAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
-}
-
-func (c *Cluster) VolumeAdminAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort))
-}
-
-func (c *Cluster) VolumePublicAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort))
-}
-
-func (c *Cluster) VolumeGRPCAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort))
-}
-
-// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
-func (c *Cluster) VolumeServerAddress() string {
-	return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(), c.volumeGrpcPort)
-}
-
-func (c *Cluster) MasterURL() string {
-	return "http://" + c.MasterAddress()
-}
-
-func (c *Cluster) VolumeAdminURL() string {
-	return "http://" + c.VolumeAdminAddress()
-}
-
-func (c *Cluster) VolumePublicURL() string {
-	return "http://" + c.VolumePublicAddress()
-}
-
-func (c *Cluster) BaseDir() string {
-	return c.baseDir
-}

test/volume_server/framework/cluster_dual.go

@@ -1,293 +0,0 @@
-package framework
-
-import (
-	"fmt"
-	"net"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"sync"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-type DualVolumeCluster struct {
-	testingTB testing.TB
-	profile   matrix.Profile
-
-	weedBinary string
-	baseDir    string
-	configDir  string
-	logsDir    string
-	keepLogs   bool
-
-	masterPort     int
-	masterGrpcPort int
-
-	volumePort0     int
-	volumeGrpcPort0 int
-	volumePubPort0  int
-	volumePort1     int
-	volumeGrpcPort1 int
-	volumePubPort1  int
-
-	masterCmd *exec.Cmd
-	volumeCmd0 *exec.Cmd
-	volumeCmd1 *exec.Cmd
-
-	cleanupOnce sync.Once
-}
-
-func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster {
-	t.Helper()
-
-	weedBinary, err := FindOrBuildWeedBinary()
-	if err != nil {
-		t.Fatalf("resolve weed binary: %v", err)
-	}
-
-	baseDir, keepLogs, err := newWorkDir()
-	if err != nil {
-		t.Fatalf("create temp test directory: %v", err)
-	}
-
-	configDir := filepath.Join(baseDir, "config")
-	logsDir := filepath.Join(baseDir, "logs")
-	masterDataDir := filepath.Join(baseDir, "master")
-	volumeDataDir0 := filepath.Join(baseDir, "volume0")
-	volumeDataDir1 := filepath.Join(baseDir, "volume1")
-	for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir0, volumeDataDir1} {
-		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
-			t.Fatalf("create %s: %v", dir, mkErr)
-		}
-	}
-
-	if err = writeSecurityConfig(configDir, profile); err != nil {
-		t.Fatalf("write security config: %v", err)
-	}
-
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
-	if err != nil {
-		t.Fatalf("allocate master port pair: %v", err)
-	}
-
-	ports, err := allocatePorts(6)
-	if err != nil {
-		t.Fatalf("allocate volume ports: %v", err)
-	}
-
-	c := &DualVolumeCluster{
-		testingTB:      t,
-		profile:        profile,
-		weedBinary:     weedBinary,
-		baseDir:        baseDir,
-		configDir:      configDir,
-		logsDir:        logsDir,
-		keepLogs:       keepLogs,
-		masterPort:     masterPort,
-		masterGrpcPort: masterGrpcPort,
-		volumePort0:    ports[0],
-		volumeGrpcPort0: ports[1],
-		volumePubPort0: ports[0],
-		volumePort1:    ports[2],
-		volumeGrpcPort1: ports[3],
-		volumePubPort1: ports[2],
-	}
-	if profile.SplitPublicPort {
-		c.volumePubPort0 = ports[4]
-		c.volumePubPort1 = ports[5]
-	}
-
-	if err = c.startMaster(masterDataDir); err != nil {
-		c.Stop()
-		t.Fatalf("start master: %v", err)
-	}
-	if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
-		masterLog := c.tailLog("master.log")
-		c.Stop()
-		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
-	}
-
-	if err = c.startVolume(0, volumeDataDir0); err != nil {
-		masterLog := c.tailLog("master.log")
-		c.Stop()
-		t.Fatalf("start first volume server: %v\nmaster log tail:\n%s", err, masterLog)
-	}
-	if err = c.waitForHTTP(c.VolumeAdminURL(0) + "/status"); err != nil {
-		volumeLog := c.tailLog("volume0.log")
-		c.Stop()
-		t.Fatalf("wait for first volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
-	}
-	if err = c.waitForTCP(c.VolumeGRPCAddress(0)); err != nil {
-		volumeLog := c.tailLog("volume0.log")
-		c.Stop()
-		t.Fatalf("wait for first volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
-	}
-
-	if err = c.startVolume(1, volumeDataDir1); err != nil {
-		volumeLog := c.tailLog("volume0.log")
-		c.Stop()
-		t.Fatalf("start second volume server: %v\nfirst volume log tail:\n%s", err, volumeLog)
-	}
-	if err = c.waitForHTTP(c.VolumeAdminURL(1) + "/status"); err != nil {
-		volumeLog := c.tailLog("volume1.log")
-		c.Stop()
-		t.Fatalf("wait for second volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
-	}
-	if err = c.waitForTCP(c.VolumeGRPCAddress(1)); err != nil {
-		volumeLog := c.tailLog("volume1.log")
-		c.Stop()
-		t.Fatalf("wait for second volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
-	}
-
-	t.Cleanup(func() {
-		c.Stop()
-	})
-
-	return c
-}
-
-func (c *DualVolumeCluster) Stop() {
-	if c == nil {
-		return
-	}
-	c.cleanupOnce.Do(func() {
-		stopProcess(c.volumeCmd1)
-		stopProcess(c.volumeCmd0)
-		stopProcess(c.masterCmd)
-		if !c.keepLogs && !c.testingTB.Failed() {
-			_ = os.RemoveAll(c.baseDir)
-		} else if c.baseDir != "" {
-			c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
-		}
-	})
-}
-
-func (c *DualVolumeCluster) startMaster(dataDir string) error {
-	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
-	if err != nil {
-		return err
-	}
-
-	args := []string{
-		"-config_dir=" + c.configDir,
-		"master",
-		"-ip=127.0.0.1",
-		"-port=" + strconv.Itoa(c.masterPort),
-		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
-		"-mdir=" + dataDir,
-		"-peers=none",
-		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
-		"-defaultReplication=000",
-	}
-
-	c.masterCmd = exec.Command(c.weedBinary, args...)
-	c.masterCmd.Dir = c.baseDir
-	c.masterCmd.Stdout = logFile
-	c.masterCmd.Stderr = logFile
-	return c.masterCmd.Start()
-}
-
-func (c *DualVolumeCluster) startVolume(index int, dataDir string) error {
-	logName := fmt.Sprintf("volume%d.log", index)
-	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
-	if err != nil {
-		return err
-	}
-
-	volumePort := c.volumePort0
-	volumeGrpcPort := c.volumeGrpcPort0
-	volumePubPort := c.volumePubPort0
-	if index == 1 {
-		volumePort = c.volumePort1
-		volumeGrpcPort = c.volumeGrpcPort1
-		volumePubPort = c.volumePubPort1
-	}
-
-	args := []string{
-		"-config_dir=" + c.configDir,
-		"volume",
-		"-ip=127.0.0.1",
-		"-port=" + strconv.Itoa(volumePort),
-		"-port.grpc=" + strconv.Itoa(volumeGrpcPort),
-		"-port.public=" + strconv.Itoa(volumePubPort),
-		"-dir=" + dataDir,
-		"-max=16",
-		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
-		"-readMode=" + c.profile.ReadMode,
-		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
-		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
-	}
-	if c.profile.InflightUploadTimeout > 0 {
-		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
-	}
-	if c.profile.InflightDownloadTimeout > 0 {
-		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
-	}
-
-	cmd := exec.Command(c.weedBinary, args...)
-	cmd.Dir = c.baseDir
-	cmd.Stdout = logFile
-	cmd.Stderr = logFile
-
-	if err = cmd.Start(); err != nil {
-		return err
-	}
-	if index == 1 {
-		c.volumeCmd1 = cmd
-	} else {
-		c.volumeCmd0 = cmd
-	}
-	return nil
-}
-
-func (c *DualVolumeCluster) waitForHTTP(url string) error {
-	return (&Cluster{}).waitForHTTP(url)
-}
-
-func (c *DualVolumeCluster) waitForTCP(addr string) error {
-	return (&Cluster{}).waitForTCP(addr)
-}
-
-func (c *DualVolumeCluster) tailLog(logName string) string {
-	return (&Cluster{logsDir: c.logsDir}).tailLog(logName)
-}
-
-func (c *DualVolumeCluster) MasterAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
-}
-
-func (c *DualVolumeCluster) MasterURL() string {
-	return "http://" + c.MasterAddress()
-}
-
-func (c *DualVolumeCluster) VolumeAdminAddress(index int) string {
-	if index == 1 {
-		return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort1))
-	}
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort0))
-}
-
-func (c *DualVolumeCluster) VolumePublicAddress(index int) string {
-	if index == 1 {
-		return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort1))
-	}
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort0))
-}
-
-func (c *DualVolumeCluster) VolumeGRPCAddress(index int) string {
-	if index == 1 {
-		return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort1))
-	}
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort0))
-}
-
-func (c *DualVolumeCluster) VolumeAdminURL(index int) string {
-	return "http://" + c.VolumeAdminAddress(index)
-}
-
-func (c *DualVolumeCluster) VolumePublicURL(index int) string {
-	return "http://" + c.VolumePublicAddress(index)
-}

test/volume_server/framework/cluster_with_filer.go

@@ -1,91 +0,0 @@
-package framework
-
-import (
-	"fmt"
-	"net"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-type ClusterWithFiler struct {
-	*Cluster
-
-	filerCmd      *exec.Cmd
-	filerPort     int
-	filerGrpcPort int
-}
-
-func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *ClusterWithFiler {
-	t.Helper()
-
-	baseCluster := StartSingleVolumeCluster(t, profile)
-
-	ports, err := allocatePorts(2)
-	if err != nil {
-		t.Fatalf("allocate filer ports: %v", err)
-	}
-
-	filerDataDir := filepath.Join(baseCluster.baseDir, "filer")
-	if mkErr := os.MkdirAll(filerDataDir, 0o755); mkErr != nil {
-		t.Fatalf("create filer data dir: %v", mkErr)
-	}
-
-	logFile, err := os.Create(filepath.Join(baseCluster.logsDir, "filer.log"))
-	if err != nil {
-		t.Fatalf("create filer log file: %v", err)
-	}
-
-	filerPort := ports[0]
-	filerGrpcPort := ports[1]
-	args := []string{
-		"-config_dir=" + baseCluster.configDir,
-		"filer",
-		"-master=127.0.0.1:" + strconv.Itoa(baseCluster.masterPort),
-		"-ip=127.0.0.1",
-		"-port=" + strconv.Itoa(filerPort),
-		"-port.grpc=" + strconv.Itoa(filerGrpcPort),
-		"-defaultStoreDir=" + filerDataDir,
-	}
-
-	filerCmd := exec.Command(baseCluster.weedBinary, args...)
-	filerCmd.Dir = baseCluster.baseDir
-	filerCmd.Stdout = logFile
-	filerCmd.Stderr = logFile
-	if err = filerCmd.Start(); err != nil {
-		t.Fatalf("start filer: %v", err)
-	}
-
-	if err = baseCluster.waitForTCP(net.JoinHostPort("127.0.0.1", strconv.Itoa(filerGrpcPort))); err != nil {
-		filerLogTail := baseCluster.tailLog("filer.log")
-		stopProcess(filerCmd)
-		t.Fatalf("wait for filer grpc readiness: %v\nfiler log tail:\n%s", err, filerLogTail)
-	}
-
-	t.Cleanup(func() {
-		stopProcess(filerCmd)
-	})
-
-	return &ClusterWithFiler{
-		Cluster:       baseCluster,
-		filerCmd:      filerCmd,
-		filerPort:     filerPort,
-		filerGrpcPort: filerGrpcPort,
-	}
-}
-
-func (c *ClusterWithFiler) FilerAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerPort))
-}
-
-func (c *ClusterWithFiler) FilerGRPCAddress() string {
-	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPort))
-}
-
-func (c *ClusterWithFiler) FilerServerAddress() string {
-	return fmt.Sprintf("%s.%d", c.FilerAddress(), c.filerGrpcPort)
-}

test/volume_server/framework/fault_injection.go

@@ -1,8 +0,0 @@
-package framework
-
-// Phase 0 placeholder for future fault injection utilities.
-//
-// Planned extensions:
-// - restart/kill selected processes
-// - temporary network isolation hooks
-// - master or peer outage helpers for proxy/replication branch coverage

test/volume_server/framework/grpc_client.go

@@ -1,28 +0,0 @@
-package framework
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-)
-
-func DialVolumeServer(t testing.TB, address string) (*grpc.ClientConn, volume_server_pb.VolumeServerClient) {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	conn, err := grpc.DialContext(ctx, address,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		grpc.WithBlock(),
-	)
-	if err != nil {
-		t.Fatalf("dial volume grpc %s: %v", address, err)
-	}
-
-	return conn, volume_server_pb.NewVolumeServerClient(conn)
-}

test/volume_server/framework/http_client.go

@@ -1,34 +0,0 @@
-package framework
-
-import (
-	"io"
-	"net/http"
-	"testing"
-	"time"
-)
-
-func NewHTTPClient() *http.Client {
-	return &http.Client{Timeout: 10 * time.Second}
-}
-
-func DoRequest(t testing.TB, client *http.Client, req *http.Request) *http.Response {
-	t.Helper()
-	resp, err := client.Do(req)
-	if err != nil {
-		t.Fatalf("http request %s %s: %v", req.Method, req.URL.String(), err)
-	}
-	return resp
-}
-
-func ReadAllAndClose(t testing.TB, resp *http.Response) []byte {
-	t.Helper()
-	if resp == nil {
-		return nil
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		t.Fatalf("read response body: %v", err)
-	}
-	return body
-}

test/volume_server/framework/volume_fixture.go

@@ -1,56 +0,0 @@
-package framework
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
-)
-
-func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
-		VolumeId:    volumeID,
-		Collection:  collection,
-		Replication: "000",
-		Version:     uint32(needle.GetCurrentVersion()),
-	})
-	if err != nil {
-		t.Fatalf("allocate volume %d: %v", volumeID, err)
-	}
-}
-
-func NewFileID(volumeID uint32, key uint64, cookie uint32) string {
-	return needle.NewFileId(needle.VolumeId(volumeID), key, cookie).String()
-}
-
-func UploadBytes(t testing.TB, client *http.Client, volumeURL, fid string, data []byte) *http.Response {
-	t.Helper()
-
-	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s", volumeURL, fid), bytes.NewReader(data))
-	if err != nil {
-		t.Fatalf("build upload request: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/octet-stream")
-	req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
-	return DoRequest(t, client, req)
-}
-
-func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
-	t.Helper()
-
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
-	if err != nil {
-		t.Fatalf("build read request: %v", err)
-	}
-	return DoRequest(t, client, req)
-}

test/volume_server/grpc/admin_extra_test.go

@@ -1,445 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/cluster"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestVolumeNeedleStatusForUploadedFile(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(21)
-	const needleID = uint64(778899)
-	const cookie = uint32(0xA1B2C3D4)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	client := framework.NewHTTPClient()
-	payload := []byte("needle-status-payload")
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
-		VolumeId: volumeID,
-		NeedleId: needleID,
-	})
-	if err != nil {
-		t.Fatalf("VolumeNeedleStatus failed: %v", err)
-	}
-	if statusResp.GetNeedleId() != needleID {
-		t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID)
-	}
-	if statusResp.GetCookie() != cookie {
-		t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie)
-	}
-	if statusResp.GetSize() == 0 {
-		t.Fatalf("expected non-zero needle size")
-	}
-}
-
-func TestVolumeNeedleStatusViaEcShardsWhenNormalVolumeUnmounted(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(26)
-	const needleID = uint64(778900)
-	const cookie = uint32(0xA1B2C3D5)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	payload := []byte("needle-status-ec-path-payload")
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("VolumeUnmount failed: %v", err)
-	}
-
-	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
-		VolumeId: volumeID,
-		NeedleId: needleID,
-	})
-	if err != nil {
-		t.Fatalf("VolumeNeedleStatus via EC shards failed: %v", err)
-	}
-	if statusResp.GetNeedleId() != needleID {
-		t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID)
-	}
-	if statusResp.GetCookie() != cookie {
-		t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie)
-	}
-	if statusResp.GetSize() == 0 {
-		t.Fatalf("expected non-zero needle size from EC-backed needle status")
-	}
-
-	_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
-		VolumeId: volumeID,
-		NeedleId: needleID + 999999,
-	})
-	if err == nil || !strings.Contains(strings.ToLower(err.Error()), "not found") {
-		t.Fatalf("VolumeNeedleStatus via EC shards missing-needle error mismatch: %v", err)
-	}
-}
-
-func TestVolumeNeedleStatusMissingVolumeAndNeedle(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(25)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
-		VolumeId: 99925,
-		NeedleId: 1,
-	})
-	if err == nil {
-		t.Fatalf("VolumeNeedleStatus should fail for missing volume")
-	}
-	if !strings.Contains(strings.ToLower(err.Error()), "volume not found") {
-		t.Fatalf("VolumeNeedleStatus missing-volume error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
-		VolumeId: volumeID,
-		NeedleId: 123456789,
-	})
-	if err == nil {
-		t.Fatalf("VolumeNeedleStatus should fail for missing needle")
-	}
-	if !strings.Contains(strings.ToLower(err.Error()), "not found") {
-		t.Fatalf("VolumeNeedleStatus missing-needle error mismatch: %v", err)
-	}
-}
-
-func mustNewRequest(t testing.TB, method, url string) *http.Request {
-	t.Helper()
-	req, err := http.NewRequest(method, url, nil)
-	if err != nil {
-		t.Fatalf("create request %s %s: %v", method, url, err)
-	}
-	return req
-}
-
-func TestVolumeConfigureInvalidReplication(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(22)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	resp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
-		VolumeId:    volumeID,
-		Replication: "bad-replication",
-	})
-	if err != nil {
-		t.Fatalf("VolumeConfigure returned grpc error: %v", err)
-	}
-	if resp.GetError() == "" {
-		t.Fatalf("VolumeConfigure expected response error for invalid replication")
-	}
-	if !strings.Contains(strings.ToLower(resp.GetError()), "replication") {
-		t.Fatalf("VolumeConfigure error should mention replication, got: %q", resp.GetError())
-	}
-}
-
-func TestVolumeConfigureSuccessAndMissingRollbackPath(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(24)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	successResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
-		VolumeId:    volumeID,
-		Replication: "000",
-	})
-	if err != nil {
-		t.Fatalf("VolumeConfigure success path returned grpc error: %v", err)
-	}
-	if successResp.GetError() != "" {
-		t.Fatalf("VolumeConfigure success path expected empty response error, got: %q", successResp.GetError())
-	}
-
-	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeStatus after successful configure failed: %v", err)
-	}
-	if statusResp.GetIsReadOnly() {
-		t.Fatalf("VolumeStatus after configure expected writable volume")
-	}
-
-	missingResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
-		VolumeId:    99024,
-		Replication: "000",
-	})
-	if err != nil {
-		t.Fatalf("VolumeConfigure missing-volume branch should return response error, got grpc error: %v", err)
-	}
-	if missingResp.GetError() == "" {
-		t.Fatalf("VolumeConfigure missing-volume expected non-empty response error")
-	}
-	lower := strings.ToLower(missingResp.GetError())
-	if !strings.Contains(lower, "not found on disk") {
-		t.Fatalf("VolumeConfigure missing-volume error should mention not found on disk, got: %q", missingResp.GetError())
-	}
-	if !strings.Contains(lower, "failed to restore mount") {
-		t.Fatalf("VolumeConfigure missing-volume error should include remount rollback failure, got: %q", missingResp.GetError())
-	}
-}
-
-func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
-		TargetType: cluster.VolumeServerType,
-		Target:     clusterHarness.VolumeServerAddress(),
-	})
-	if err != nil {
-		t.Fatalf("Ping target volume server failed: %v", err)
-	}
-	if pingResp.GetRemoteTimeNs() == 0 {
-		t.Fatalf("expected remote timestamp from ping target volume server")
-	}
-
-	if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
-		t.Fatalf("VolumeServerLeave failed: %v", err)
-	}
-
-	client := framework.NewHTTPClient()
-	healthURL := clusterHarness.VolumeAdminURL() + "/healthz"
-	deadline := time.Now().Add(5 * time.Second)
-	for {
-		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL))
-		_ = framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode == http.StatusServiceUnavailable {
-			return
-		}
-		if time.Now().After(deadline) {
-			t.Fatalf("expected healthz to return 503 after leave, got %d", resp.StatusCode)
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-}
-
-func TestVolumeServerLeaveIsIdempotent(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
-		t.Fatalf("first VolumeServerLeave failed: %v", err)
-	}
-	if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
-		t.Fatalf("second VolumeServerLeave should be idempotent success, got: %v", err)
-	}
-
-	client := framework.NewHTTPClient()
-	healthURL := clusterHarness.VolumeAdminURL() + "/healthz"
-	deadline := time.Now().Add(5 * time.Second)
-	for {
-		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL))
-		_ = framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode == http.StatusServiceUnavailable {
-			return
-		}
-		if time.Now().After(deadline) {
-			t.Fatalf("expected healthz to stay 503 after repeated leave, got %d", resp.StatusCode)
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-}
-
-func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	unknownResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
-		TargetType: "unknown-type",
-		Target:     "127.0.0.1:12345",
-	})
-	if err != nil {
-		t.Fatalf("Ping unknown target type should not return grpc error, got: %v", err)
-	}
-	if unknownResp.GetRemoteTimeNs() != 0 {
-		t.Fatalf("Ping unknown target type expected remote_time_ns=0, got %d", unknownResp.GetRemoteTimeNs())
-	}
-	if unknownResp.GetStopTimeNs() < unknownResp.GetStartTimeNs() {
-		t.Fatalf("Ping unknown target type expected stop_time_ns >= start_time_ns")
-	}
-
-	_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
-		TargetType: cluster.MasterType,
-		Target:     "127.0.0.1:1",
-	})
-	if err == nil {
-		t.Fatalf("Ping master target should fail when target is unreachable")
-	}
-	if !strings.Contains(err.Error(), "ping master") {
-		t.Fatalf("Ping master unreachable error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
-		TargetType: cluster.FilerType,
-		Target:     "127.0.0.1:1",
-	})
-	if err == nil {
-		t.Fatalf("Ping filer target should fail when target is unreachable")
-	}
-	if !strings.Contains(err.Error(), "ping filer") {
-		t.Fatalf("Ping filer unreachable error mismatch: %v", err)
-	}
-}
-
-func TestPingMasterTargetSuccess(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
-		TargetType: cluster.MasterType,
-		Target:     clusterHarness.MasterAddress(),
-	})
-	if err != nil {
-		t.Fatalf("Ping master target success path failed: %v", err)
-	}
-	if resp.GetRemoteTimeNs() == 0 {
-		t.Fatalf("Ping master target expected non-zero remote time")
-	}
-	if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
-		t.Fatalf("Ping master target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
-	}
-}
-
-func TestPingFilerTargetSuccess(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
-		TargetType: cluster.FilerType,
-		Target:     clusterHarness.FilerServerAddress(),
-	})
-	if err != nil {
-		t.Fatalf("Ping filer target success path failed: %v", err)
-	}
-	if resp.GetRemoteTimeNs() == 0 {
-		t.Fatalf("Ping filer target expected non-zero remote time")
-	}
-	if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
-		t.Fatalf("Ping filer target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
-	}
-}

test/volume_server/grpc/admin_lifecycle_test.go

@@ -1,215 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-)
-
-func TestVolumeAdminLifecycleRPCs(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	const volumeID = uint32(11)
-	framework.AllocateVolume(t, client, volumeID, "")
-
-	statusResp, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeStatus failed: %v", err)
-	}
-	if statusResp.GetFileCount() != 0 {
-		t.Fatalf("new volume should be empty, got file_count=%d", statusResp.GetFileCount())
-	}
-
-	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
-		t.Fatalf("VolumeUnmount failed: %v", err)
-	}
-	if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil {
-		t.Fatalf("VolumeMount failed: %v", err)
-	}
-
-	if _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil {
-		t.Fatalf("VolumeDelete failed: %v", err)
-	}
-
-	_, err = client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err == nil {
-		t.Fatalf("VolumeStatus should fail after delete")
-	}
-	if st, ok := status.FromError(err); !ok || st.Code() == codes.OK {
-		t.Fatalf("VolumeStatus error should be a non-OK grpc status, got: %v", err)
-	}
-}
-
-func TestVolumeDeleteOnlyEmptyVariants(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(13)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 66001, 0x11223344)
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("volume-delete-only-empty"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true})
-	if err == nil || !strings.Contains(err.Error(), "volume not empty") {
-		t.Fatalf("VolumeDelete only_empty=true expected volume-not-empty error, got: %v", err)
-	}
-
-	_, err = grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: false})
-	if err != nil {
-		t.Fatalf("VolumeDelete only_empty=false failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err == nil {
-		t.Fatalf("VolumeStatus should fail after non-empty delete with only_empty=false")
-	}
-}
-
-func TestMaintenanceModeRejectsAllocateVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{VolumeId: 12, Replication: "000"})
-	if err == nil {
-		t.Fatalf("AllocateVolume should fail when maintenance mode is enabled")
-	}
-	if !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("expected maintenance mode error, got: %v", err)
-	}
-}
-
-func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	const missingVolumeID = uint32(99331)
-	const volumeID = uint32(14)
-
-	if _, err := client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: missingVolumeID}); err != nil {
-		t.Fatalf("VolumeUnmount missing volume should be idempotent success, got: %v", err)
-	}
-
-	_, err := client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: missingVolumeID})
-	if err == nil {
-		t.Fatalf("VolumeMount missing volume should fail")
-	}
-	if !strings.Contains(err.Error(), "not found on disk") {
-		t.Fatalf("VolumeMount missing volume error mismatch: %v", err)
-	}
-
-	framework.AllocateVolume(t, client, volumeID, "")
-
-	_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
-		VolumeId:    volumeID,
-		Replication: "000",
-	})
-	if err == nil {
-		t.Fatalf("AllocateVolume duplicate should fail")
-	}
-	if !strings.Contains(strings.ToLower(err.Error()), "already exists") {
-		t.Fatalf("AllocateVolume duplicate error mismatch: %v", err)
-	}
-
-	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
-		t.Fatalf("VolumeUnmount existing volume failed: %v", err)
-	}
-	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
-		t.Fatalf("VolumeUnmount already-unmounted volume should be idempotent success, got: %v", err)
-	}
-	if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil {
-		t.Fatalf("VolumeMount remount failed: %v", err)
-	}
-}
-
-func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(15)
-	framework.AllocateVolume(t, client, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	_, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true})
-	if err == nil {
-		t.Fatalf("VolumeDelete should fail when maintenance mode is enabled")
-	}
-	if !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("expected maintenance mode error, got: %v", err)
-	}
-}

test/volume_server/grpc/admin_readonly_collection_test.go

@@ -1,177 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(72)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
-		VolumeId: volumeID,
-		Persist:  false,
-	})
-	if err != nil {
-		t.Fatalf("VolumeMarkReadonly failed: %v", err)
-	}
-
-	readOnlyStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeStatus after readonly failed: %v", err)
-	}
-	if !readOnlyStatus.GetIsReadOnly() {
-		t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly")
-	}
-
-	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeMarkWritable failed: %v", err)
-	}
-
-	writableStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeStatus after writable failed: %v", err)
-	}
-	if writableStatus.GetIsReadOnly() {
-		t.Fatalf("VolumeStatus expected readonly=false after VolumeMarkWritable")
-	}
-}
-
-func TestVolumeMarkReadonlyPersistTrue(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(74)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
-		VolumeId: volumeID,
-		Persist:  true,
-	})
-	if err != nil {
-		t.Fatalf("VolumeMarkReadonly persist=true failed: %v", err)
-	}
-
-	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeStatus after persist readonly failed: %v", err)
-	}
-	if !statusResp.GetIsReadOnly() {
-		t.Fatalf("VolumeStatus expected readonly=true after persist readonly")
-	}
-}
-
-func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true})
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98772})
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err)
-	}
-
-	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{
-			Maintenance: true,
-			Version:     stateResp.GetState().GetVersion(),
-		},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 1, Persist: true})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeMarkReadonly maintenance error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 1})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err)
-	}
-}
-
-func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(73)
-	const collection = "it-delete-collection"
-
-	framework.AllocateVolume(t, grpcClient, volumeID, collection)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeStatus before DeleteCollection failed: %v", err)
-	}
-
-	_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
-	if err != nil {
-		t.Fatalf("DeleteCollection existing collection failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
-	if err == nil {
-		t.Fatalf("VolumeStatus should fail after collection delete")
-	}
-	if !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("VolumeStatus after DeleteCollection error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
-	if err != nil {
-		t.Fatalf("DeleteCollection idempotent retry failed: %v", err)
-	}
-}

test/volume_server/grpc/batch_delete_test.go

@@ -1,264 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"bytes"
-	"context"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestBatchDeleteInvalidFidAndMaintenanceMode(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"bad-fid"}})
-	if err != nil {
-		t.Fatalf("BatchDelete invalid fid should return response, got error: %v", err)
-	}
-	if len(resp.GetResults()) != 1 {
-		t.Fatalf("expected one batch delete result, got %d", len(resp.GetResults()))
-	}
-	if got := resp.GetResults()[0].GetStatus(); got != 400 {
-		t.Fatalf("invalid fid expected status 400, got %d", got)
-	}
-
-	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	_, err = client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"1,1234567890ab"}})
-	if err == nil {
-		t.Fatalf("BatchDelete should fail when maintenance mode is enabled")
-	}
-	if !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("expected maintenance mode error, got: %v", err)
-	}
-}
-
-func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(31)
-	const needleID = uint64(900001)
-	const correctCookie = uint32(0x1122AABB)
-	const wrongCookie = uint32(0x1122AABC)
-	framework.AllocateVolume(t, client, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, needleID, correctCookie)
-	uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-cookie-check"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	mismatchResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
-		FileIds:         []string{wrongCookieFid},
-		SkipCookieCheck: false,
-	})
-	if err != nil {
-		t.Fatalf("BatchDelete with cookie check failed: %v", err)
-	}
-	if len(mismatchResp.GetResults()) != 1 {
-		t.Fatalf("BatchDelete cookie mismatch expected 1 result, got %d", len(mismatchResp.GetResults()))
-	}
-	if mismatchResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
-		t.Fatalf("BatchDelete cookie mismatch expected status 400, got %d", mismatchResp.GetResults()[0].GetStatus())
-	}
-
-	skipCheckResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
-		FileIds:         []string{wrongCookieFid},
-		SkipCookieCheck: true,
-	})
-	if err != nil {
-		t.Fatalf("BatchDelete skip cookie check failed: %v", err)
-	}
-	if len(skipCheckResp.GetResults()) != 1 {
-		t.Fatalf("BatchDelete skip check expected 1 result, got %d", len(skipCheckResp.GetResults()))
-	}
-	if skipCheckResp.GetResults()[0].GetStatus() != http.StatusAccepted {
-		t.Fatalf("BatchDelete skip check expected status 202, got %d", skipCheckResp.GetResults()[0].GetStatus())
-	}
-
-	readAfterDelete := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
-	_ = framework.ReadAllAndClose(t, readAfterDelete)
-	if readAfterDelete.StatusCode != http.StatusNotFound {
-		t.Fatalf("read after skip-check batch delete expected 404, got %d", readAfterDelete.StatusCode)
-	}
-}
-
-func TestBatchDeleteMixedStatusesAndMismatchStopsProcessing(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(32)
-	framework.AllocateVolume(t, client, volumeID, "")
-
-	const needleA = uint64(910001)
-	const needleB = uint64(910002)
-	const needleC = uint64(910003)
-	const cookieA = uint32(0x11111111)
-	const cookieB = uint32(0x22222222)
-	const cookieC = uint32(0x33333333)
-
-	httpClient := framework.NewHTTPClient()
-	fidA := framework.NewFileID(volumeID, needleA, cookieA)
-	fidB := framework.NewFileID(volumeID, needleB, cookieB)
-	fidC := framework.NewFileID(volumeID, needleC, cookieC)
-
-	for _, tc := range []struct {
-		fid  string
-		body string
-	}{
-		{fid: fidA, body: "batch-delete-mixed-a"},
-		{fid: fidB, body: "batch-delete-mixed-b"},
-		{fid: fidC, body: "batch-delete-mixed-c"},
-	} {
-		uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, []byte(tc.body))
-		_ = framework.ReadAllAndClose(t, uploadResp)
-		if uploadResp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload %s expected 201, got %d", tc.fid, uploadResp.StatusCode)
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	missingFid := framework.NewFileID(volumeID, 919999, 0x44444444)
-	mixedResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
-		FileIds: []string{"bad-fid", fidA, missingFid},
-	})
-	if err != nil {
-		t.Fatalf("BatchDelete mixed status request failed: %v", err)
-	}
-	if len(mixedResp.GetResults()) != 3 {
-		t.Fatalf("BatchDelete mixed status expected 3 results, got %d", len(mixedResp.GetResults()))
-	}
-	if mixedResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
-		t.Fatalf("BatchDelete mixed result[0] expected 400, got %d", mixedResp.GetResults()[0].GetStatus())
-	}
-	if mixedResp.GetResults()[1].GetStatus() != http.StatusAccepted {
-		t.Fatalf("BatchDelete mixed result[1] expected 202, got %d", mixedResp.GetResults()[1].GetStatus())
-	}
-	if mixedResp.GetResults()[2].GetStatus() != http.StatusNotFound {
-		t.Fatalf("BatchDelete mixed result[2] expected 404, got %d", mixedResp.GetResults()[2].GetStatus())
-	}
-
-	readDeletedA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
-	_ = framework.ReadAllAndClose(t, readDeletedA)
-	if readDeletedA.StatusCode != http.StatusNotFound {
-		t.Fatalf("fidA should be deleted after batch delete, got status %d", readDeletedA.StatusCode)
-	}
-
-	wrongCookieB := framework.NewFileID(volumeID, needleB, cookieB+1)
-	stopResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
-		FileIds: []string{wrongCookieB, fidC},
-	})
-	if err != nil {
-		t.Fatalf("BatchDelete mismatch-stop request failed: %v", err)
-	}
-	if len(stopResp.GetResults()) != 1 {
-		t.Fatalf("BatchDelete mismatch-stop expected 1 result due early break, got %d", len(stopResp.GetResults()))
-	}
-	if stopResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
-		t.Fatalf("BatchDelete mismatch-stop expected 400, got %d", stopResp.GetResults()[0].GetStatus())
-	}
-
-	readB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
-	_ = framework.ReadAllAndClose(t, readB)
-	if readB.StatusCode != http.StatusOK {
-		t.Fatalf("fidB should remain after cookie mismatch path, got %d", readB.StatusCode)
-	}
-
-	readC := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidC)
-	_ = framework.ReadAllAndClose(t, readC)
-	if readC.StatusCode != http.StatusOK {
-		t.Fatalf("fidC should remain when batch processing stops on mismatch, got %d", readC.StatusCode)
-	}
-}
-
-func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(33)
-	framework.AllocateVolume(t, client, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 920001, 0x5555AAAA)
-	req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid+"?cm=true", bytes.NewReader([]byte("manifest-placeholder-payload")))
-	if err != nil {
-		t.Fatalf("create chunk manifest upload request: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/octet-stream")
-	uploadResp := framework.DoRequest(t, httpClient, req)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("chunk manifest upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}})
-	if err != nil {
-		t.Fatalf("BatchDelete chunk manifest should return response, got grpc error: %v", err)
-	}
-	if len(resp.GetResults()) != 1 {
-		t.Fatalf("BatchDelete chunk manifest expected one result, got %d", len(resp.GetResults()))
-	}
-	if resp.GetResults()[0].GetStatus() != http.StatusNotAcceptable {
-		t.Fatalf("BatchDelete chunk manifest expected status 406, got %d", resp.GetResults()[0].GetStatus())
-	}
-	if !strings.Contains(resp.GetResults()[0].GetError(), "ChunkManifest") {
-		t.Fatalf("BatchDelete chunk manifest expected error mentioning ChunkManifest, got %q", resp.GetResults()[0].GetError())
-	}
-
-	readResp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
-	_ = framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusOK {
-		t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode)
-	}
-}

test/volume_server/grpc/copy_receive_variants_test.go

@@ -1,431 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"io"
-	"math"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestVolumeIncrementalCopyDataAndNoDataPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(91)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 770001, 0x1122AABB)
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("incremental-copy-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != 201 {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	dataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{
-		VolumeId: volumeID,
-		SinceNs:  0,
-	})
-	if err != nil {
-		t.Fatalf("VolumeIncrementalCopy start failed: %v", err)
-	}
-
-	totalBytes := 0
-	for {
-		msg, recvErr := dataStream.Recv()
-		if recvErr == io.EOF {
-			break
-		}
-		if recvErr != nil {
-			t.Fatalf("VolumeIncrementalCopy recv failed: %v", recvErr)
-		}
-		totalBytes += len(msg.GetFileContent())
-	}
-	if totalBytes == 0 {
-		t.Fatalf("VolumeIncrementalCopy expected streamed bytes for since_ns=0")
-	}
-
-	noDataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{
-		VolumeId: volumeID,
-		SinceNs:  math.MaxUint64,
-	})
-	if err != nil {
-		t.Fatalf("VolumeIncrementalCopy no-data start failed: %v", err)
-	}
-	_, err = noDataStream.Recv()
-	if err != io.EOF {
-		t.Fatalf("VolumeIncrementalCopy no-data expected EOF, got: %v", err)
-	}
-}
-
-func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(92)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	missingNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:                 volumeID,
-		Ext:                      ".definitely-missing",
-		CompactionRevision:       math.MaxUint32,
-		StopOffset:               1,
-		IgnoreSourceFileNotFound: false,
-	})
-	if err == nil {
-		_, err = missingNoIgnore.Recv()
-	}
-	if err == nil {
-		t.Fatalf("CopyFile should fail for missing source file when ignore_source_file_not_found=false")
-	}
-
-	missingIgnored, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:                 volumeID,
-		Ext:                      ".definitely-missing",
-		CompactionRevision:       math.MaxUint32,
-		StopOffset:               1,
-		IgnoreSourceFileNotFound: true,
-	})
-	if err != nil {
-		t.Fatalf("CopyFile ignore-not-found start failed: %v", err)
-	}
-	_, err = missingIgnored.Recv()
-	if err != io.EOF {
-		t.Fatalf("CopyFile ignore-not-found expected EOF, got: %v", err)
-	}
-
-	stopZeroStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:                 volumeID,
-		Ext:                      ".definitely-missing",
-		CompactionRevision:       math.MaxUint32,
-		StopOffset:               0,
-		IgnoreSourceFileNotFound: false,
-	})
-	if err != nil {
-		t.Fatalf("CopyFile stop_offset=0 start failed: %v", err)
-	}
-	_, err = stopZeroStream.Recv()
-	if err != io.EOF {
-		t.Fatalf("CopyFile stop_offset=0 expected EOF, got: %v", err)
-	}
-}
-
-func TestCopyFileCompactionRevisionMismatch(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(94)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:           volumeID,
-		Ext:                ".idx",
-		CompactionRevision: 1, // fresh volume starts at revision 0
-		StopOffset:         1,
-	})
-	if err == nil {
-		_, err = stream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "is compacted") {
-		t.Fatalf("CopyFile compaction mismatch error mismatch: %v", err)
-	}
-}
-
-func TestReceiveFileProtocolViolationResponses(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	contentFirstStream, err := grpcClient.ReceiveFile(ctx)
-	if err != nil {
-		t.Fatalf("ReceiveFile stream create failed: %v", err)
-	}
-	if err = contentFirstStream.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_FileContent{
-			FileContent: []byte("content-before-info"),
-		},
-	}); err != nil {
-		t.Fatalf("ReceiveFile send content-first failed: %v", err)
-	}
-	contentFirstResp, err := contentFirstStream.CloseAndRecv()
-	if err != nil {
-		t.Fatalf("ReceiveFile content-first close failed: %v", err)
-	}
-	if !strings.Contains(contentFirstResp.GetError(), "file info must be sent first") {
-		t.Fatalf("ReceiveFile content-first response mismatch: %+v", contentFirstResp)
-	}
-
-	unknownTypeStream, err := grpcClient.ReceiveFile(ctx)
-	if err != nil {
-		t.Fatalf("ReceiveFile stream create for unknown-type failed: %v", err)
-	}
-	if err = unknownTypeStream.Send(&volume_server_pb.ReceiveFileRequest{}); err != nil {
-		t.Fatalf("ReceiveFile send unknown-type request failed: %v", err)
-	}
-	unknownTypeResp, err := unknownTypeStream.CloseAndRecv()
-	if err != nil {
-		t.Fatalf("ReceiveFile unknown-type close failed: %v", err)
-	}
-	if !strings.Contains(unknownTypeResp.GetError(), "unknown message type") {
-		t.Fatalf("ReceiveFile unknown-type response mismatch: %+v", unknownTypeResp)
-	}
-}
-
-func TestReceiveFileSuccessForRegularVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(95)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	payloadA := []byte("receive-file-chunk-a:")
-	payloadB := []byte("receive-file-chunk-b")
-	expected := append(append([]byte{}, payloadA...), payloadB...)
-
-	receiveStream, err := grpcClient.ReceiveFile(ctx)
-	if err != nil {
-		t.Fatalf("ReceiveFile stream create failed: %v", err)
-	}
-
-	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_Info{
-			Info: &volume_server_pb.ReceiveFileInfo{
-				VolumeId:   volumeID,
-				Ext:        ".tmprecv",
-				Collection: "",
-				IsEcVolume: false,
-				FileSize:   uint64(len(expected)),
-			},
-		},
-	}); err != nil {
-		t.Fatalf("ReceiveFile send info failed: %v", err)
-	}
-	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA},
-	}); err != nil {
-		t.Fatalf("ReceiveFile send payloadA failed: %v", err)
-	}
-	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB},
-	}); err != nil {
-		t.Fatalf("ReceiveFile send payloadB failed: %v", err)
-	}
-
-	resp, err := receiveStream.CloseAndRecv()
-	if err != nil {
-		t.Fatalf("ReceiveFile close failed: %v", err)
-	}
-	if resp.GetError() != "" {
-		t.Fatalf("ReceiveFile unexpected error response: %+v", resp)
-	}
-	if resp.GetBytesWritten() != uint64(len(expected)) {
-		t.Fatalf("ReceiveFile bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected))
-	}
-
-	copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:           volumeID,
-		Ext:                ".tmprecv",
-		CompactionRevision: math.MaxUint32,
-		StopOffset:         uint64(len(expected)),
-	})
-	if err != nil {
-		t.Fatalf("CopyFile for received data start failed: %v", err)
-	}
-
-	var copied []byte
-	for {
-		msg, recvErr := copyStream.Recv()
-		if recvErr == io.EOF {
-			break
-		}
-		if recvErr != nil {
-			t.Fatalf("CopyFile for received data recv failed: %v", recvErr)
-		}
-		copied = append(copied, msg.GetFileContent()...)
-	}
-
-	if string(copied) != string(expected) {
-		t.Fatalf("received file data mismatch: got %q want %q", string(copied), string(expected))
-	}
-}
-
-func TestReceiveFileSuccessForEcVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	const volumeID = uint32(96)
-	const collection = "ec-receive-success"
-	const ext = ".ec00"
-
-	payloadA := []byte("receive-ec-file-chunk-a:")
-	payloadB := []byte("receive-ec-file-chunk-b")
-	expected := append(append([]byte{}, payloadA...), payloadB...)
-
-	receiveStream, err := grpcClient.ReceiveFile(ctx)
-	if err != nil {
-		t.Fatalf("ReceiveFile stream create failed: %v", err)
-	}
-
-	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_Info{
-			Info: &volume_server_pb.ReceiveFileInfo{
-				VolumeId:   volumeID,
-				Ext:        ext,
-				Collection: collection,
-				IsEcVolume: true,
-				ShardId:    0,
-				FileSize:   uint64(len(expected)),
-			},
-		},
-	}); err != nil {
-		t.Fatalf("ReceiveFile send EC info failed: %v", err)
-	}
-	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA},
-	}); err != nil {
-		t.Fatalf("ReceiveFile send EC payloadA failed: %v", err)
-	}
-	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB},
-	}); err != nil {
-		t.Fatalf("ReceiveFile send EC payloadB failed: %v", err)
-	}
-
-	resp, err := receiveStream.CloseAndRecv()
-	if err != nil {
-		t.Fatalf("ReceiveFile EC close failed: %v", err)
-	}
-	if resp.GetError() != "" {
-		t.Fatalf("ReceiveFile EC unexpected error response: %+v", resp)
-	}
-	if resp.GetBytesWritten() != uint64(len(expected)) {
-		t.Fatalf("ReceiveFile EC bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected))
-	}
-
-	copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:           volumeID,
-		Collection:         collection,
-		IsEcVolume:         true,
-		Ext:                ext,
-		CompactionRevision: math.MaxUint32,
-		StopOffset:         uint64(len(expected)),
-	})
-	if err != nil {
-		t.Fatalf("CopyFile for received EC data start failed: %v", err)
-	}
-
-	var copied []byte
-	for {
-		msg, recvErr := copyStream.Recv()
-		if recvErr == io.EOF {
-			break
-		}
-		if recvErr != nil {
-			t.Fatalf("CopyFile for received EC data recv failed: %v", recvErr)
-		}
-		copied = append(copied, msg.GetFileContent()...)
-	}
-
-	if string(copied) != string(expected) {
-		t.Fatalf("received EC file data mismatch: got %q want %q", string(copied), string(expected))
-	}
-}
-
-func TestCopyFileEcVolumeIgnoreMissingSourcePaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	streamNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:                 99601,
-		Collection:               "ec-copy-missing",
-		IsEcVolume:               true,
-		Ext:                      ".ec00",
-		CompactionRevision:       math.MaxUint32,
-		StopOffset:               1,
-		IgnoreSourceFileNotFound: false,
-	})
-	if err == nil {
-		_, err = streamNoIgnore.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found ec volume id") {
-		t.Fatalf("CopyFile EC missing source error mismatch: %v", err)
-	}
-
-	streamIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:                 99602,
-		Collection:               "ec-copy-missing",
-		IsEcVolume:               true,
-		Ext:                      ".ec00",
-		CompactionRevision:       math.MaxUint32,
-		StopOffset:               1,
-		IgnoreSourceFileNotFound: true,
-	})
-	if err != nil {
-		t.Fatalf("CopyFile EC ignore-missing start failed: %v", err)
-	}
-	_, err = streamIgnore.Recv()
-	if err != io.EOF {
-		t.Fatalf("CopyFile EC ignore-missing expected EOF, got: %v", err)
-	}
-}

test/volume_server/grpc/copy_sync_test.go

@@ -1,284 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"io"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(41)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	syncResp, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VolumeSyncStatus failed: %v", err)
-	}
-	if syncResp.GetVolumeId() != volumeID {
-		t.Fatalf("VolumeSyncStatus volume id mismatch: got %d want %d", syncResp.GetVolumeId(), volumeID)
-	}
-
-	statusResp, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus failed: %v", err)
-	}
-	if statusResp.GetVolumeId() != volumeID {
-		t.Fatalf("ReadVolumeFileStatus volume id mismatch: got %d want %d", statusResp.GetVolumeId(), volumeID)
-	}
-	if statusResp.GetVersion() == 0 {
-		t.Fatalf("ReadVolumeFileStatus expected non-zero version")
-	}
-}
-
-func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: 98761})
-	if err == nil {
-		t.Fatalf("VolumeSyncStatus should fail for missing volume")
-	}
-
-	incrementalStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{VolumeId: 98762, SinceNs: 0})
-	if err == nil {
-		_, err = incrementalStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("VolumeIncrementalCopy missing-volume error mismatch: %v", err)
-	}
-
-	readAllStream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{98763}})
-	if err == nil {
-		_, err = readAllStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", err)
-	}
-
-	copyFileStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{VolumeId: 98764, Ext: ".dat", StopOffset: 1})
-	if err == nil {
-		_, err = copyFileStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("CopyFile missing-volume error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: 98765})
-	if err == nil || !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("ReadVolumeFileStatus missing-volume error mismatch: %v", err)
-	}
-}
-
-func TestVolumeCopyAndReceiveFileMaintenanceRejection(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	copyStream, err := grpcClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{VolumeId: 1, SourceDataNode: "127.0.0.1:1234"})
-	if err == nil {
-		_, err = copyStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeCopy maintenance error mismatch: %v", err)
-	}
-
-	receiveClient, err := grpcClient.ReceiveFile(ctx)
-	if err != nil {
-		t.Fatalf("ReceiveFile client creation failed: %v", err)
-	}
-	_ = receiveClient.Send(&volume_server_pb.ReceiveFileRequest{
-		Data: &volume_server_pb.ReceiveFileRequest_Info{
-			Info: &volume_server_pb.ReceiveFileInfo{VolumeId: 1, Ext: ".dat"},
-		},
-	})
-	_, err = receiveClient.CloseAndRecv()
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("ReceiveFile maintenance error mismatch: %v", err)
-	}
-}
-
-func TestVolumeCopySuccessFromPeerAndMountsDestination(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
-	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer sourceConn.Close()
-	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
-	defer destConn.Close()
-
-	const volumeID = uint32(42)
-	framework.AllocateVolume(t, sourceClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 880001, 0x12345678)
-	payload := []byte("volume-copy-success-payload")
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload to source expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
-		VolumeId:       volumeID,
-		Collection:     "",
-		SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
-	})
-	if err != nil {
-		t.Fatalf("VolumeCopy start failed: %v", err)
-	}
-
-	sawFinalAppendTimestamp := false
-	for {
-		msg, recvErr := copyStream.Recv()
-		if recvErr == io.EOF {
-			break
-		}
-		if recvErr != nil {
-			t.Fatalf("VolumeCopy recv failed: %v", recvErr)
-		}
-		if msg.GetLastAppendAtNs() > 0 {
-			sawFinalAppendTimestamp = true
-		}
-	}
-	if !sawFinalAppendTimestamp {
-		t.Fatalf("VolumeCopy expected final response with last_append_at_ns")
-	}
-
-	destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
-	destReadBody := framework.ReadAllAndClose(t, destReadResp)
-	if destReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("read from copied destination expected 200, got %d", destReadResp.StatusCode)
-	}
-	if string(destReadBody) != string(payload) {
-		t.Fatalf("destination copied payload mismatch: got %q want %q", string(destReadBody), string(payload))
-	}
-}
-
-func TestVolumeCopyOverwritesExistingDestinationVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
-	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer sourceConn.Close()
-	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
-	defer destConn.Close()
-
-	const volumeID = uint32(43)
-	framework.AllocateVolume(t, sourceClient, volumeID, "")
-	framework.AllocateVolume(t, destClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 880002, 0x23456789)
-	sourcePayload := []byte("volume-copy-overwrite-source")
-	destPayload := []byte("volume-copy-overwrite-destination-old")
-
-	sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, sourcePayload)
-	_ = framework.ReadAllAndClose(t, sourceUploadResp)
-	if sourceUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload to source expected 201, got %d", sourceUploadResp.StatusCode)
-	}
-
-	destUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid, destPayload)
-	_ = framework.ReadAllAndClose(t, destUploadResp)
-	if destUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload to destination expected 201, got %d", destUploadResp.StatusCode)
-	}
-
-	destReadBeforeResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
-	destReadBeforeBody := framework.ReadAllAndClose(t, destReadBeforeResp)
-	if destReadBeforeResp.StatusCode != http.StatusOK {
-		t.Fatalf("destination pre-copy read expected 200, got %d", destReadBeforeResp.StatusCode)
-	}
-	if string(destReadBeforeBody) != string(destPayload) {
-		t.Fatalf("destination pre-copy payload mismatch: got %q want %q", string(destReadBeforeBody), string(destPayload))
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
-		VolumeId:       volumeID,
-		Collection:     "",
-		SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
-	})
-	if err != nil {
-		t.Fatalf("VolumeCopy overwrite start failed: %v", err)
-	}
-
-	sawFinalAppendTimestamp := false
-	for {
-		msg, recvErr := copyStream.Recv()
-		if recvErr == io.EOF {
-			break
-		}
-		if recvErr != nil {
-			t.Fatalf("VolumeCopy overwrite recv failed: %v", recvErr)
-		}
-		if msg.GetLastAppendAtNs() > 0 {
-			sawFinalAppendTimestamp = true
-		}
-	}
-	if !sawFinalAppendTimestamp {
-		t.Fatalf("VolumeCopy overwrite expected final response with last_append_at_ns")
-	}
-
-	destReadAfterResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
-	destReadAfterBody := framework.ReadAllAndClose(t, destReadAfterResp)
-	if destReadAfterResp.StatusCode != http.StatusOK {
-		t.Fatalf("destination post-copy read expected 200, got %d", destReadAfterResp.StatusCode)
-	}
-	if string(destReadAfterBody) != string(sourcePayload) {
-		t.Fatalf("destination post-copy payload mismatch: got %q want %q", string(destReadAfterBody), string(sourcePayload))
-	}
-}

test/volume_server/grpc/data_rw_test.go

@@ -1,146 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestReadNeedleBlobAndMetaMissingVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
-		VolumeId: 99111,
-		Offset:   0,
-		Size:     16,
-	})
-	if err == nil {
-		t.Fatalf("ReadNeedleBlob should fail for missing volume")
-	}
-	if !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("ReadNeedleBlob missing volume error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
-		VolumeId: 99112,
-		NeedleId: 1,
-		Offset:   0,
-		Size:     16,
-	})
-	if err == nil {
-		t.Fatalf("ReadNeedleMeta should fail for missing volume")
-	}
-	if !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("ReadNeedleMeta missing volume error mismatch: %v", err)
-	}
-}
-
-func TestWriteNeedleBlobMaintenanceAndMissingVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
-		VolumeId:   99113,
-		NeedleId:   1,
-		NeedleBlob: []byte("abc"),
-		Size:       3,
-	})
-	if err == nil {
-		t.Fatalf("WriteNeedleBlob should fail for missing volume")
-	}
-	if !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("WriteNeedleBlob missing volume error mismatch: %v", err)
-	}
-
-	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
-		VolumeId:   1,
-		NeedleId:   2,
-		NeedleBlob: []byte("def"),
-		Size:       3,
-	})
-	if err == nil {
-		t.Fatalf("WriteNeedleBlob should fail in maintenance mode")
-	}
-	if !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("WriteNeedleBlob maintenance mode error mismatch: %v", err)
-	}
-}
-
-func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(92)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 880001, 0xCCDD1122)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("invalid-offset-check"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != 201 {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
-		VolumeId: volumeID,
-		Offset:   1 << 40,
-		Size:     64,
-	})
-	if err == nil {
-		t.Fatalf("ReadNeedleBlob should fail for invalid offset")
-	}
-	if !strings.Contains(strings.ToLower(err.Error()), "read needle blob") {
-		t.Fatalf("ReadNeedleBlob invalid offset error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
-		VolumeId: volumeID,
-		NeedleId: 880001,
-		Offset:   1 << 40,
-		Size:     64,
-	})
-	if err == nil {
-		t.Fatalf("ReadNeedleMeta should fail for invalid offset")
-	}
-}

test/volume_server/grpc/data_stream_success_test.go

@@ -1,273 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"io"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
-	"github.com/seaweedfs/seaweedfs/weed/storage/types"
-)
-
-func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(83)
-	const sourceNeedleID = uint64(333333)
-	const sourceCookie = uint32(0xABCD0102)
-	const clonedNeedleID = uint64(333334)
-
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	payload := []byte("blob-roundtrip-content")
-	fid := framework.NewFileID(volumeID, sourceNeedleID, sourceCookie)
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != 201 {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("ReadVolumeFileStatus failed: %v", err)
-	}
-	if fileStatus.GetIdxFileSize() == 0 {
-		t.Fatalf("expected non-zero idx file size after upload")
-	}
-
-	idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
-		VolumeId:           volumeID,
-		Ext:                ".idx",
-		CompactionRevision: fileStatus.GetCompactionRevision(),
-		StopOffset:         fileStatus.GetIdxFileSize(),
-	})
-	offset, size := findNeedleOffsetAndSize(t, idxBytes, sourceNeedleID)
-
-	blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
-		VolumeId: volumeID,
-		Offset:   offset,
-		Size:     size,
-	})
-	if err != nil {
-		t.Fatalf("ReadNeedleBlob failed: %v", err)
-	}
-	if len(blobResp.GetNeedleBlob()) == 0 {
-		t.Fatalf("ReadNeedleBlob returned empty blob")
-	}
-
-	metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
-		VolumeId: volumeID,
-		NeedleId: sourceNeedleID,
-		Offset:   offset,
-		Size:     size,
-	})
-	if err != nil {
-		t.Fatalf("ReadNeedleMeta failed: %v", err)
-	}
-	if metaResp.GetCookie() != sourceCookie {
-		t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), sourceCookie)
-	}
-
-	_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
-		VolumeId:   volumeID,
-		NeedleId:   clonedNeedleID,
-		Size:       size,
-		NeedleBlob: blobResp.GetNeedleBlob(),
-	})
-	if err != nil {
-		t.Fatalf("WriteNeedleBlob failed: %v", err)
-	}
-
-	clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
-		VolumeId: volumeID,
-		NeedleId: clonedNeedleID,
-	})
-	if err != nil {
-		t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err)
-	}
-	if clonedStatus.GetNeedleId() != sourceNeedleID {
-		t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID)
-	}
-	if clonedStatus.GetCookie() != sourceCookie {
-		t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie)
-	}
-
-	clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie))
-	clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp)
-	if clonedReadResp.StatusCode != 200 {
-		t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode)
-	}
-	if string(clonedReadBody) != string(payload) {
-		t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload))
-	}
-}
-
-func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(84)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	expected := map[uint64]string{
-		444441: "read-all-needle-one",
-		444442: "read-all-needle-two",
-	}
-	for key, body := range expected {
-		resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, key, 0xA0B0C0D0), []byte(body))
-		_ = framework.ReadAllAndClose(t, resp)
-		if resp.StatusCode != 201 {
-			t.Fatalf("upload for key %d expected 201, got %d", key, resp.StatusCode)
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}})
-	if err != nil {
-		t.Fatalf("ReadAllNeedles start failed: %v", err)
-	}
-
-	seen := map[uint64]string{}
-	for {
-		msg, recvErr := stream.Recv()
-		if recvErr == io.EOF {
-			break
-		}
-		if recvErr != nil {
-			t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
-		}
-		if _, wanted := expected[msg.GetNeedleId()]; wanted {
-			seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
-		}
-	}
-
-	for key, body := range expected {
-		got, found := seen[key]
-		if !found {
-			t.Fatalf("ReadAllNeedles missing key %d in stream", key)
-		}
-		if got != body {
-			t.Fatalf("ReadAllNeedles body mismatch for key %d: got %q want %q", key, got, body)
-		}
-	}
-}
-
-func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const existingVolumeID = uint32(85)
-	const missingVolumeID = uint32(98585)
-	const needleID = uint64(445551)
-	framework.AllocateVolume(t, grpcClient, existingVolumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(existingVolumeID, needleID, 0xAA11BB22)
-	payload := "read-all-existing-then-missing"
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte(payload))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != 201 {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
-		VolumeIds: []uint32{existingVolumeID, missingVolumeID},
-	})
-	if err != nil {
-		t.Fatalf("ReadAllNeedles start failed: %v", err)
-	}
-
-	seenUploadedNeedle := false
-	for {
-		msg, recvErr := stream.Recv()
-		if recvErr == io.EOF {
-			t.Fatalf("ReadAllNeedles expected stream error for missing volume, got EOF")
-		}
-		if recvErr != nil {
-			if !strings.Contains(recvErr.Error(), "not found volume id") {
-				t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", recvErr)
-			}
-			break
-		}
-		if msg.GetNeedleId() == needleID && string(msg.GetNeedleBlob()) == payload {
-			seenUploadedNeedle = true
-		}
-	}
-
-	if !seenUploadedNeedle {
-		t.Fatalf("ReadAllNeedles should stream entries from existing volume before missing-volume abort")
-	}
-}
-
-func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.CopyFile(ctx, req)
-	if err != nil {
-		t.Fatalf("CopyFile start failed: %v", err)
-	}
-
-	var out []byte
-	for {
-		msg, recvErr := stream.Recv()
-		if recvErr == io.EOF {
-			return out
-		}
-		if recvErr != nil {
-			t.Fatalf("CopyFile recv failed: %v", recvErr)
-		}
-		out = append(out, msg.GetFileContent()...)
-	}
-}
-
-func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
-	t.Helper()
-
-	for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
-		key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
-		if uint64(key) != needleID {
-			continue
-		}
-		if entryOffset.IsZero() || entrySize <= 0 {
-			continue
-		}
-		return entryOffset.ToActualOffset(), int32(entrySize)
-	}
-
-	t.Fatalf("needle id %d not found in idx entries", needleID)
-	return 0, 0
-}

test/volume_server/grpc/erasure_coding_test.go

@@ -1,777 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"io"
-	"math"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
-	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-)
-
-func TestEcMaintenanceModeRejections(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{
-			Maintenance: true,
-			Version:     stateResp.GetState().GetVersion(),
-		},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
-		VolumeId:       1,
-		Collection:     "",
-		SourceDataNode: "127.0.0.1:1",
-		ShardIds:       []uint32{0},
-	})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
-		VolumeId:   1,
-		Collection: "",
-		ShardIds:   []uint32{0},
-	})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeEcShardsDelete maintenance error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
-		VolumeId:   1,
-		Collection: "",
-		FileKey:    1,
-		Version:    3,
-	})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeEcBlobDelete maintenance error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
-		VolumeId:   1,
-		Collection: "",
-	})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeEcShardsToVolume maintenance error mismatch: %v", err)
-	}
-}
-
-func TestEcMissingInvalidAndNoopPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   98791,
-		Collection: "",
-	})
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeEcShardsGenerate missing-volume error mismatch: %v", err)
-	}
-
-	rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
-		VolumeId:   98792,
-		Collection: "ec-rebuild",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsRebuild missing-volume should return empty success, got: %v", err)
-	}
-	if len(rebuildResp.GetRebuiltShardIds()) != 0 {
-		t.Fatalf("VolumeEcShardsRebuild expected no rebuilt shards for missing volume, got %v", rebuildResp.GetRebuiltShardIds())
-	}
-
-	_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
-		VolumeId:       98793,
-		Collection:     "ec-copy",
-		SourceDataNode: "127.0.0.1:1",
-		ShardIds:       []uint32{0},
-		DiskId:         99,
-	})
-	if err == nil || !strings.Contains(err.Error(), "invalid disk_id") {
-		t.Fatalf("VolumeEcShardsCopy invalid-disk error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
-		VolumeId:   98794,
-		Collection: "ec-delete",
-		ShardIds:   []uint32{0, 1},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsDelete missing-volume should be no-op success, got: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId:   98795,
-		Collection: "ec-mount",
-		ShardIds:   []uint32{0},
-	})
-	if err == nil {
-		t.Fatalf("VolumeEcShardsMount should fail for missing EC shards")
-	}
-
-	_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
-		VolumeId: 98796,
-		ShardIds: []uint32{0},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsUnmount missing shards should be no-op success, got: %v", err)
-	}
-
-	readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
-		VolumeId: 98797,
-		ShardId:  0,
-		Offset:   0,
-		Size:     1,
-	})
-	if err == nil {
-		_, err = readStream.Recv()
-	}
-	if err == nil || err == io.EOF {
-		t.Fatalf("VolumeEcShardRead should fail for missing EC volume")
-	}
-
-	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
-		VolumeId:   98798,
-		Collection: "ec-blob",
-		FileKey:    1,
-		Version:    3,
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcBlobDelete missing local EC volume should be no-op success, got: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
-		VolumeId:   98799,
-		Collection: "ec-to-volume",
-	})
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeEcShardsToVolume missing-volume error mismatch: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
-		VolumeId: 98800,
-	})
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeEcShardsInfo missing-volume error mismatch: %v", err)
-	}
-}
-
-func TestEcGenerateMountInfoUnmountLifecycle(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(115)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 990001, 0x1234ABCD)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-generate-lifecycle-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate success path failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount success path failed: %v", err)
-	}
-
-	infoResp, err := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
-		VolumeId: volumeID,
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsInfo after mount failed: %v", err)
-	}
-	if len(infoResp.GetEcShardInfos()) == 0 {
-		t.Fatalf("VolumeEcShardsInfo expected non-empty shard infos after mount")
-	}
-	if infoResp.GetVolumeSize() == 0 {
-		t.Fatalf("VolumeEcShardsInfo expected non-zero volume size after mount")
-	}
-
-	_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
-		VolumeId: volumeID,
-		ShardIds: []uint32{0},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsUnmount success path failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
-		VolumeId: volumeID,
-	})
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeEcShardsInfo after unmount expected not-found error, got: %v", err)
-	}
-}
-
-func TestEcShardReadAndBlobDeleteLifecycle(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(116)
-	const fileKey = uint64(990002)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, fileKey, 0x2233CCDD)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-shard-read-delete-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount failed: %v", err)
-	}
-
-	readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
-		VolumeId: volumeID,
-		ShardId:  0,
-		Offset:   0,
-		Size:     1,
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardRead start failed: %v", err)
-	}
-	firstChunk, err := readStream.Recv()
-	if err != nil {
-		t.Fatalf("VolumeEcShardRead recv failed: %v", err)
-	}
-	if len(firstChunk.GetData()) == 0 {
-		t.Fatalf("VolumeEcShardRead expected non-empty data chunk before deletion")
-	}
-
-	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		FileKey:    fileKey,
-		Version:    uint32(needle.GetCurrentVersion()),
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcBlobDelete first delete failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		FileKey:    fileKey,
-		Version:    uint32(needle.GetCurrentVersion()),
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcBlobDelete second delete should be idempotent success, got: %v", err)
-	}
-
-	deletedStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
-		VolumeId: volumeID,
-		ShardId:  0,
-		FileKey:  fileKey,
-		Offset:   0,
-		Size:     1,
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err)
-	}
-	deletedMsg, err := deletedStream.Recv()
-	if err != nil {
-		t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err)
-	}
-	if !deletedMsg.GetIsDeleted() {
-		t.Fatalf("VolumeEcShardRead expected IsDeleted=true after blob delete")
-	}
-	_, err = deletedStream.Recv()
-	if err != io.EOF {
-		t.Fatalf("VolumeEcShardRead deleted-check expected EOF after deleted marker, got: %v", err)
-	}
-}
-
-func TestEcRebuildMissingShardLifecycle(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(117)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 990003, 0x3344DDEE)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-rebuild-shard-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0},
-	})
-	if err == nil {
-		t.Fatalf("VolumeEcShardsMount should fail when shard 0 has been deleted")
-	}
-
-	rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsRebuild failed: %v", err)
-	}
-	if len(rebuildResp.GetRebuiltShardIds()) == 0 {
-		t.Fatalf("VolumeEcShardsRebuild expected rebuilt shard ids")
-	}
-	foundShard0 := false
-	for _, shardID := range rebuildResp.GetRebuiltShardIds() {
-		if shardID == 0 {
-			foundShard0 = true
-			break
-		}
-	}
-	if !foundShard0 {
-		t.Fatalf("VolumeEcShardsRebuild expected shard 0 to be rebuilt, got %v", rebuildResp.GetRebuiltShardIds())
-	}
-
-	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount shard 0 after rebuild failed: %v", err)
-	}
-}
-
-func TestEcShardsToVolumeMissingShardAndNoLiveEntries(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	httpClient := framework.NewHTTPClient()
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	t.Run("missing shard returns error", func(t *testing.T) {
-		const volumeID = uint32(118)
-		framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-		fid := framework.NewFileID(volumeID, 990004, 0x4455EEFF)
-		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-to-volume-missing-shard-content"))
-		_ = framework.ReadAllAndClose(t, uploadResp)
-		if uploadResp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-		}
-
-		_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-			VolumeId:   volumeID,
-			Collection: "",
-		})
-		if err != nil {
-			t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
-		}
-
-		_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
-			VolumeId:   volumeID,
-			Collection: "",
-			ShardIds:   []uint32{0},
-		})
-		if err != nil {
-			t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err)
-		}
-
-		_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-			VolumeId:   volumeID,
-			Collection: "",
-			ShardIds:   []uint32{1},
-		})
-		if err != nil {
-			t.Fatalf("VolumeEcShardsMount shard 1 failed: %v", err)
-		}
-
-		_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
-			VolumeId:   volumeID,
-			Collection: "",
-		})
-		if err == nil || !strings.Contains(err.Error(), "missing shard 0") {
-			t.Fatalf("VolumeEcShardsToVolume missing-shard error mismatch: %v", err)
-		}
-	})
-
-	t.Run("no live entries returns failed precondition", func(t *testing.T) {
-		const volumeID = uint32(119)
-		const needleID = uint64(990005)
-		const cookie = uint32(0x5566FF11)
-		framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-		fid := framework.NewFileID(volumeID, needleID, cookie)
-		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-no-live-entries-content"))
-		_ = framework.ReadAllAndClose(t, uploadResp)
-		if uploadResp.StatusCode != http.StatusCreated {
-			t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-		}
-
-		deleteResp := framework.DoRequest(t, httpClient, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid))
-		_ = framework.ReadAllAndClose(t, deleteResp)
-		if deleteResp.StatusCode != http.StatusAccepted {
-			t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
-		}
-
-		_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-			VolumeId:   volumeID,
-			Collection: "",
-		})
-		if err != nil {
-			t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
-		}
-
-		_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-			VolumeId:   volumeID,
-			Collection: "",
-			ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
-		})
-		if err != nil {
-			t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
-		}
-
-		_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
-			VolumeId:   volumeID,
-			Collection: "",
-		})
-		if err == nil {
-			t.Fatalf("VolumeEcShardsToVolume expected failed-precondition error when no live entries")
-		}
-		if status.Code(err) != codes.FailedPrecondition {
-			t.Fatalf("VolumeEcShardsToVolume no-live-entries expected FailedPrecondition, got %v (%v)", status.Code(err), err)
-		}
-		if !strings.Contains(err.Error(), erasure_coding.EcNoLiveEntriesSubstring) {
-			t.Fatalf("VolumeEcShardsToVolume no-live-entries error should mention %q, got %v", erasure_coding.EcNoLiveEntriesSubstring, err)
-		}
-	})
-}
-
-func TestEcShardsToVolumeSuccessRoundTrip(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(120)
-	const needleID = uint64(990006)
-	const cookie = uint32(0x66771122)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	payload := []byte("ec-shards-to-volume-success-roundtrip-content")
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsToVolume success path failed: %v", err)
-	}
-
-	readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid)
-	readBody := framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusOK {
-		t.Fatalf("post-conversion read expected 200, got %d", readResp.StatusCode)
-	}
-	if string(readBody) != string(payload) {
-		t.Fatalf("post-conversion payload mismatch: got %q want %q", string(readBody), string(payload))
-	}
-}
-
-func TestEcShardsDeleteLastShardRemovesEcx(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(121)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 990007, 0x77882233)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-delete-all-shards-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
-	}
-
-	// Verify .ecx is present before deleting all shards.
-	ecxBeforeDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:           volumeID,
-		Collection:         "",
-		IsEcVolume:         true,
-		Ext:                ".ecx",
-		CompactionRevision: math.MaxUint32,
-		StopOffset:         1,
-	})
-	if err != nil {
-		t.Fatalf("CopyFile .ecx before shard deletion start failed: %v", err)
-	}
-	if _, err = ecxBeforeDelete.Recv(); err != nil {
-		t.Fatalf("CopyFile .ecx before shard deletion recv failed: %v", err)
-	}
-
-	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13},
-	})
-	if err != nil {
-		t.Fatalf("VolumeEcShardsDelete all shards failed: %v", err)
-	}
-
-	ecxAfterDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-		VolumeId:           volumeID,
-		Collection:         "",
-		IsEcVolume:         true,
-		Ext:                ".ecx",
-		CompactionRevision: math.MaxUint32,
-		StopOffset:         1,
-	})
-	if err == nil {
-		_, err = ecxAfterDelete.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found ec volume id") {
-		t.Fatalf("CopyFile .ecx after deleting all shards should fail not-found, got: %v", err)
-	}
-}
-
-func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
-	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer sourceConn.Close()
-	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
-	defer destConn.Close()
-
-	const volumeID = uint32(122)
-	framework.AllocateVolume(t, sourceClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 990008, 0x88993344)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, []byte("ec-copy-from-peer-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("source upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := sourceClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
-		VolumeId:   volumeID,
-		Collection: "",
-	})
-	if err != nil {
-		t.Fatalf("source VolumeEcShardsGenerate failed: %v", err)
-	}
-
-	sourceDataNode := clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1]
-	_, err = destClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
-		VolumeId:       volumeID,
-		Collection:     "",
-		SourceDataNode: sourceDataNode,
-		ShardIds:       []uint32{0},
-		CopyEcxFile:    true,
-		CopyVifFile:    true,
-	})
-	if err != nil {
-		t.Fatalf("destination VolumeEcShardsCopy success path failed: %v", err)
-	}
-
-	for _, ext := range []string{".ec00", ".ecx", ".vif"} {
-		copyStream, copyErr := destClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
-			VolumeId:           volumeID,
-			Collection:         "",
-			IsEcVolume:         true,
-			Ext:                ext,
-			CompactionRevision: math.MaxUint32,
-			StopOffset:         1,
-		})
-		if copyErr != nil {
-			t.Fatalf("destination CopyFile %s start failed: %v", ext, copyErr)
-		}
-		if _, copyErr = copyStream.Recv(); copyErr != nil {
-			t.Fatalf("destination CopyFile %s recv failed: %v", ext, copyErr)
-		}
-	}
-}
-
-func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
-		VolumeId:       12345,
-		Collection:     "",
-		SourceDataNode: "127.0.0.1:1.1",
-		ShardIds:       []uint32{0},
-		CopyEcxFile:    true,
-	})
-	if err == nil || !strings.Contains(err.Error(), "VolumeEcShardsCopy volume") {
-		t.Fatalf("VolumeEcShardsCopy source-unavailable error mismatch: %v", err)
-	}
-}

test/volume_server/grpc/health_state_test.go

@@ -1,139 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestStateAndStatusRPCs(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	if initialState.GetState() == nil {
-		t.Fatalf("GetState returned nil state")
-	}
-
-	setResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{
-			Maintenance: true,
-			Version:     initialState.GetState().GetVersion(),
-		},
-	})
-	if err != nil {
-		t.Fatalf("SetState(maintenance=true) failed: %v", err)
-	}
-	if !setResp.GetState().GetMaintenance() {
-		t.Fatalf("expected maintenance=true after SetState")
-	}
-
-	setResp, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{
-			Maintenance: false,
-			Version:     setResp.GetState().GetVersion(),
-		},
-	})
-	if err != nil {
-		t.Fatalf("SetState(maintenance=false) failed: %v", err)
-	}
-	if setResp.GetState().GetMaintenance() {
-		t.Fatalf("expected maintenance=false after SetState")
-	}
-
-	statusResp, err := client.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{})
-	if err != nil {
-		t.Fatalf("VolumeServerStatus failed: %v", err)
-	}
-	if statusResp.GetVersion() == "" {
-		t.Fatalf("VolumeServerStatus returned empty version")
-	}
-	if len(statusResp.GetDiskStatuses()) == 0 {
-		t.Fatalf("VolumeServerStatus returned no disk statuses")
-	}
-	if statusResp.GetState() == nil {
-		t.Fatalf("VolumeServerStatus returned nil state")
-	}
-	if statusResp.GetMemoryStatus() == nil {
-		t.Fatalf("VolumeServerStatus returned nil memory status")
-	}
-	if statusResp.GetMemoryStatus().GetGoroutines() <= 0 {
-		t.Fatalf("VolumeServerStatus memory status should report goroutines, got %d", statusResp.GetMemoryStatus().GetGoroutines())
-	}
-
-	pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
-	if err != nil {
-		t.Fatalf("Ping failed: %v", err)
-	}
-	if pingResp.GetStartTimeNs() == 0 || pingResp.GetStopTimeNs() == 0 {
-		t.Fatalf("Ping timestamps should be non-zero: %+v", pingResp)
-	}
-	if pingResp.GetStopTimeNs() < pingResp.GetStartTimeNs() {
-		t.Fatalf("Ping stop time should be >= start time: %+v", pingResp)
-	}
-}
-
-func TestSetStateVersionMismatchAndNilStateNoop(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	initialVersion := initialState.GetState().GetVersion()
-
-	staleResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{
-			Maintenance: true,
-			Version:     initialVersion + 1,
-		},
-	})
-	if err == nil {
-		t.Fatalf("SetState with stale version should fail")
-	}
-	if !strings.Contains(err.Error(), "version mismatch") {
-		t.Fatalf("SetState stale version error mismatch: %v", err)
-	}
-	if staleResp.GetState().GetVersion() != initialVersion {
-		t.Fatalf("SetState stale version should not mutate server version: got %d want %d", staleResp.GetState().GetVersion(), initialVersion)
-	}
-	if staleResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() {
-		t.Fatalf("SetState stale version should not mutate maintenance flag")
-	}
-
-	nilResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{})
-	if err != nil {
-		t.Fatalf("SetState nil-state request should be no-op success: %v", err)
-	}
-	if nilResp.GetState().GetVersion() != initialVersion {
-		t.Fatalf("SetState nil-state should keep version unchanged: got %d want %d", nilResp.GetState().GetVersion(), initialVersion)
-	}
-	if nilResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() {
-		t.Fatalf("SetState nil-state should keep maintenance unchanged")
-	}
-}

test/volume_server/grpc/scrub_query_test.go

@@ -1,387 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"io"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestScrubVolumeIndexAndUnsupportedMode(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(61)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	indexResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
-		VolumeIds: []uint32{volumeID},
-		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
-	})
-	if err != nil {
-		t.Fatalf("ScrubVolume index mode failed: %v", err)
-	}
-	if indexResp.GetTotalVolumes() != 1 {
-		t.Fatalf("ScrubVolume expected total_volumes=1, got %d", indexResp.GetTotalVolumes())
-	}
-
-	_, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
-		VolumeIds: []uint32{volumeID},
-		Mode:      volume_server_pb.VolumeScrubMode(99),
-	})
-	if err == nil {
-		t.Fatalf("ScrubVolume should fail for unsupported mode")
-	}
-	if !strings.Contains(err.Error(), "unsupported volume scrub mode") {
-		t.Fatalf("ScrubVolume unsupported mode error mismatch: %v", err)
-	}
-}
-
-func TestScrubEcVolumeMissingVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
-		VolumeIds: []uint32{98765},
-		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
-	})
-	if err == nil {
-		t.Fatalf("ScrubEcVolume should fail for missing EC volume")
-	}
-	if !strings.Contains(err.Error(), "EC volume id") {
-		t.Fatalf("ScrubEcVolume missing-volume error mismatch: %v", err)
-	}
-}
-
-func TestScrubEcVolumeAutoSelectNoEcVolumes(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
-		Mode: volume_server_pb.VolumeScrubMode_INDEX,
-	})
-	if err != nil {
-		t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
-	}
-	if resp.GetTotalVolumes() != 0 {
-		t.Fatalf("ScrubEcVolume auto-select expected total_volumes=0 without EC data, got %d", resp.GetTotalVolumes())
-	}
-	if len(resp.GetBrokenVolumeIds()) != 0 {
-		t.Fatalf("ScrubEcVolume auto-select expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
-	}
-}
-
-func TestQueryInvalidAndMissingFileIDPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	invalidStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
-		FromFileIds: []string{"bad-fid"},
-		Selections:  []string{"name"},
-		Filter:      &volume_server_pb.QueryRequest_Filter{},
-		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
-			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{},
-		},
-	})
-	if err == nil {
-		_, err = invalidStream.Recv()
-	}
-	if err == nil {
-		t.Fatalf("Query should fail for invalid file id")
-	}
-
-	missingFid := framework.NewFileID(98766, 1, 1)
-	missingStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
-		FromFileIds: []string{missingFid},
-		Selections:  []string{"name"},
-		Filter:      &volume_server_pb.QueryRequest_Filter{},
-		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
-			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{},
-		},
-	})
-	if err == nil {
-		_, err = missingStream.Recv()
-	}
-	if err == nil {
-		t.Fatalf("Query should fail for missing file id volume")
-	}
-}
-
-func TestScrubVolumeAutoSelectAndAllModes(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeIDA = uint32(62)
-	const volumeIDB = uint32(63)
-	framework.AllocateVolume(t, grpcClient, volumeIDA, "")
-	framework.AllocateVolume(t, grpcClient, volumeIDB, "")
-
-	// upload some data so index files are not zero-sized
-	httpClient := framework.NewHTTPClient()
-	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeIDA, 1, 1), []byte("test data A"))
-	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeIDB, 2, 2), []byte("test data B"))
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	autoResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
-		Mode: volume_server_pb.VolumeScrubMode_INDEX,
-	})
-	if err != nil {
-		t.Fatalf("ScrubVolume auto-select failed: %v", err)
-	}
-	if autoResp.GetTotalVolumes() < 2 {
-		t.Fatalf("ScrubVolume auto-select expected at least 2 volumes, got %d", autoResp.GetTotalVolumes())
-	}
-
-	localResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
-		VolumeIds: []uint32{volumeIDA},
-		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
-	})
-	if err != nil {
-		t.Fatalf("ScrubVolume local mode failed: %v", err)
-	}
-	if localResp.GetTotalVolumes() != 1 {
-		t.Fatalf("ScrubVolume local mode expected total_volumes=1, got %d", localResp.GetTotalVolumes())
-	}
-	if len(localResp.GetBrokenVolumeIds()) != 0 {
-		t.Fatalf("ScrubVolume local mode expected no broken volumes, got %v: %v", localResp.GetBrokenVolumeIds(), localResp.GetDetails())
-	}
-
-	fullResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
-		VolumeIds: []uint32{volumeIDA},
-		Mode:      volume_server_pb.VolumeScrubMode_FULL,
-	})
-	if err != nil {
-		t.Fatalf("ScrubVolume full mode failed: %v", err)
-	}
-	if fullResp.GetTotalVolumes() != 1 {
-		t.Fatalf("ScrubVolume full mode expected total_volumes=1, got %d", fullResp.GetTotalVolumes())
-	}
-	if len(fullResp.GetBrokenVolumeIds()) != 0 {
-		t.Fatalf("ScrubVolume full mode expected no broken volumes, got %v: %v", fullResp.GetBrokenVolumeIds(), fullResp.GetDetails())
-	}
-}
-
-func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(64)
-	const needleID = uint64(777001)
-	const cookie = uint32(0xAABBCCDD)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	jsonLines := []byte("{\"score\":3}\n{\"score\":12}\n{\"score\":18}\n")
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != 201 {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
-		FromFileIds: []string{fid},
-		Selections:  []string{"score"},
-		Filter: &volume_server_pb.QueryRequest_Filter{
-			Field:   "score",
-			Operand: ">",
-			Value:   "10",
-		},
-		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
-			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
-		},
-	})
-	if err != nil {
-		t.Fatalf("Query json start failed: %v", err)
-	}
-
-	firstStripe, err := queryStream.Recv()
-	if err != nil {
-		t.Fatalf("Query json recv failed: %v", err)
-	}
-	records := string(firstStripe.GetRecords())
-	if !strings.Contains(records, "score:12") || !strings.Contains(records, "score:18") {
-		t.Fatalf("Query json records missing expected filtered scores: %q", records)
-	}
-	if strings.Contains(records, "score:3") {
-		t.Fatalf("Query json records should not include filtered-out score: %q", records)
-	}
-	_, err = queryStream.Recv()
-	if err != io.EOF {
-		t.Fatalf("Query json expected EOF after first stripe, got: %v", err)
-	}
-
-	csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
-		FromFileIds: []string{fid},
-		Selections:  []string{"score"},
-		Filter:      &volume_server_pb.QueryRequest_Filter{},
-		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
-			CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{},
-		},
-	})
-	if err != nil {
-		t.Fatalf("Query csv start failed: %v", err)
-	}
-	_, err = csvStream.Recv()
-	if err != io.EOF {
-		t.Fatalf("Query csv expected EOF with no rows, got: %v", err)
-	}
-}
-
-func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(65)
-	const needleID = uint64(777002)
-	const cookie = uint32(0xABABCDCD)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	jsonLines := []byte("{\"score\":1}\n{\"score\":2}\n")
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != 201 {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
-		FromFileIds: []string{fid},
-		Selections:  []string{"score"},
-		Filter: &volume_server_pb.QueryRequest_Filter{
-			Field:   "score",
-			Operand: ">",
-			Value:   "100",
-		},
-		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
-			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
-		},
-	})
-	if err != nil {
-		t.Fatalf("Query json no-match start failed: %v", err)
-	}
-
-	firstStripe, err := queryStream.Recv()
-	if err != nil {
-		t.Fatalf("Query json no-match recv failed: %v", err)
-	}
-	if len(firstStripe.GetRecords()) != 0 {
-		t.Fatalf("Query json no-match expected empty records stripe, got: %q", string(firstStripe.GetRecords()))
-	}
-
-	_, err = queryStream.Recv()
-	if err != io.EOF {
-		t.Fatalf("Query json no-match expected EOF after first empty stripe, got: %v", err)
-	}
-}
-
-func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(66)
-	const needleID = uint64(777003)
-	const cookie = uint32(0xCDCDABAB)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	jsonLines := []byte("{\"score\":7}\n{\"score\":8}\n")
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != 201 {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
-		FromFileIds: []string{wrongCookieFid},
-		Selections:  []string{"score"},
-		Filter: &volume_server_pb.QueryRequest_Filter{
-			Field:   "score",
-			Operand: ">",
-			Value:   "0",
-		},
-		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
-			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
-		},
-	})
-	if err != nil {
-		t.Fatalf("Query start for cookie mismatch should not fail immediately, got: %v", err)
-	}
-
-	_, err = stream.Recv()
-	if err != io.EOF {
-		t.Fatalf("Query cookie mismatch expected EOF with no streamed records, got: %v", err)
-	}
-}

test/volume_server/grpc/tail_test.go

@@ -1,206 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestVolumeTailSenderMissingVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{VolumeId: 77777, SinceNs: 0, IdleTimeoutSeconds: 1})
-	if err == nil {
-		_, err = stream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found volume") {
-		t.Fatalf("VolumeTailSender missing-volume error mismatch: %v", err)
-	}
-}
-
-func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(71)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
-		VolumeId:           volumeID,
-		SinceNs:            0,
-		IdleTimeoutSeconds: 1,
-	})
-	if err != nil {
-		t.Fatalf("VolumeTailSender start failed: %v", err)
-	}
-
-	msg, err := stream.Recv()
-	if err != nil {
-		t.Fatalf("VolumeTailSender first recv failed: %v", err)
-	}
-	if !msg.GetIsLastChunk() {
-		t.Fatalf("expected first tail message to be heartbeat IsLastChunk=true")
-	}
-
-	_, err = stream.Recv()
-	if err != io.EOF {
-		t.Fatalf("expected EOF after idle timeout drain, got: %v", err)
-	}
-}
-
-func TestVolumeTailReceiverMissingVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{VolumeId: 88888, SourceVolumeServer: clusterHarness.VolumeServerAddress(), SinceNs: 0, IdleTimeoutSeconds: 1})
-	if err == nil || !strings.Contains(err.Error(), "receiver not found volume") {
-		t.Fatalf("VolumeTailReceiver missing-volume error mismatch: %v", err)
-	}
-}
-
-func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
-	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer sourceConn.Close()
-	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
-	defer destConn.Close()
-
-	const volumeID = uint32(72)
-	framework.AllocateVolume(t, sourceClient, volumeID, "")
-	framework.AllocateVolume(t, destClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 880003, 0x3456789A)
-	payload := []byte("tail-receiver-replicates-source-updates")
-
-	sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload)
-	_ = framework.ReadAllAndClose(t, sourceUploadResp)
-	if sourceUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("source upload expected 201, got %d", sourceUploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	_, err := destClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
-		VolumeId:           volumeID,
-		SourceVolumeServer: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
-		SinceNs:            0,
-		IdleTimeoutSeconds: 1,
-	})
-	if err != nil {
-		t.Fatalf("VolumeTailReceiver success path failed: %v", err)
-	}
-
-	destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
-	destReadBody := framework.ReadAllAndClose(t, destReadResp)
-	if destReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("destination read after tail receive expected 200, got %d", destReadResp.StatusCode)
-	}
-	if string(destReadBody) != string(payload) {
-		t.Fatalf("destination tail-received payload mismatch: got %q want %q", string(destReadBody), string(payload))
-	}
-}
-
-func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(73)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	httpClient := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 880004, 0x456789AB)
-	largePayload := bytes.Repeat([]byte("L"), 2*1024*1024+128*1024)
-	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
-		VolumeId:           volumeID,
-		SinceNs:            0,
-		IdleTimeoutSeconds: 1,
-	})
-	if err != nil {
-		t.Fatalf("VolumeTailSender start failed: %v", err)
-	}
-
-	dataChunkCount := 0
-	sawNonLastDataChunk := false
-	sawLastDataChunk := false
-	for {
-		msg, recvErr := stream.Recv()
-		if recvErr == io.EOF {
-			break
-		}
-		if recvErr != nil {
-			t.Fatalf("VolumeTailSender recv failed: %v", recvErr)
-		}
-		if len(msg.GetNeedleBody()) == 0 {
-			continue
-		}
-		dataChunkCount++
-		if msg.GetIsLastChunk() {
-			sawLastDataChunk = true
-		} else {
-			sawNonLastDataChunk = true
-		}
-	}
-
-	if dataChunkCount < 2 {
-		t.Fatalf("VolumeTailSender expected multiple chunks for large needle, got %d", dataChunkCount)
-	}
-	if !sawNonLastDataChunk {
-		t.Fatalf("VolumeTailSender expected at least one non-last data chunk")
-	}
-	if !sawLastDataChunk {
-		t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
-	}
-}

test/volume_server/grpc/tiering_remote_test.go

@@ -1,236 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestFetchAndWriteNeedleMaintenanceAndMissingVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
-		VolumeId: 98781,
-		NeedleId: 1,
-	})
-	if err == nil || !strings.Contains(err.Error(), "not found volume id") {
-		t.Fatalf("FetchAndWriteNeedle missing-volume error mismatch: %v", err)
-	}
-
-	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{
-			Maintenance: true,
-			Version:     stateResp.GetState().GetVersion(),
-		},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	_, err = grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
-		VolumeId: 1,
-		NeedleId: 1,
-	})
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("FetchAndWriteNeedle maintenance error mismatch: %v", err)
-	}
-}
-
-func TestFetchAndWriteNeedleInvalidRemoteConfig(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(88)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
-		VolumeId: volumeID,
-		NeedleId: 1,
-		Cookie:   1,
-		Size:     1,
-		RemoteConf: &remote_pb.RemoteConf{
-			Name: "it-invalid-remote",
-			Type: "does-not-exist",
-		},
-		RemoteLocation: &remote_pb.RemoteStorageLocation{
-			Name: "it-invalid-remote",
-			Path: "/test",
-		},
-	})
-	if err == nil || !strings.Contains(err.Error(), "get remote client") {
-		t.Fatalf("FetchAndWriteNeedle invalid-remote error mismatch: %v", err)
-	}
-}
-
-func TestVolumeTierMoveDatToRemoteErrorPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(85)
-	const collection = "tier-collection"
-	framework.AllocateVolume(t, grpcClient, volumeID, collection)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	missingStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
-		VolumeId:               98782,
-		Collection:             collection,
-		DestinationBackendName: "dummy",
-	})
-	if err == nil {
-		_, err = missingStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeTierMoveDatToRemote missing-volume error mismatch: %v", err)
-	}
-
-	mismatchStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
-		VolumeId:               volumeID,
-		Collection:             "wrong-collection",
-		DestinationBackendName: "dummy",
-	})
-	if err == nil {
-		_, err = mismatchStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "unexpected input") {
-		t.Fatalf("VolumeTierMoveDatToRemote collection mismatch error mismatch: %v", err)
-	}
-
-	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{
-			Maintenance: true,
-			Version:     stateResp.GetState().GetVersion(),
-		},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	maintenanceStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
-		VolumeId:               volumeID,
-		Collection:             collection,
-		DestinationBackendName: "dummy",
-	})
-	if err == nil {
-		_, err = maintenanceStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
-		t.Fatalf("VolumeTierMoveDatToRemote maintenance error mismatch: %v", err)
-	}
-}
-
-func TestVolumeTierMoveDatToRemoteMissingBackend(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(89)
-	const collection = "tier-missing-backend"
-	framework.AllocateVolume(t, grpcClient, volumeID, collection)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
-		VolumeId:               volumeID,
-		Collection:             collection,
-		DestinationBackendName: "definitely-missing-backend",
-	})
-	if err == nil {
-		_, err = stream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "destination definitely-missing-backend not found") {
-		t.Fatalf("VolumeTierMoveDatToRemote missing-backend error mismatch: %v", err)
-	}
-}
-
-func TestVolumeTierMoveDatFromRemoteErrorPaths(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(86)
-	const collection = "tier-download-collection"
-	framework.AllocateVolume(t, grpcClient, volumeID, collection)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	missingStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
-		VolumeId:   98783,
-		Collection: collection,
-	})
-	if err == nil {
-		_, err = missingStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Fatalf("VolumeTierMoveDatFromRemote missing-volume error mismatch: %v", err)
-	}
-
-	mismatchStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
-		VolumeId:   volumeID,
-		Collection: "wrong-collection",
-	})
-	if err == nil {
-		_, err = mismatchStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "unexpected input") {
-		t.Fatalf("VolumeTierMoveDatFromRemote collection mismatch error mismatch: %v", err)
-	}
-
-	localDiskStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
-		VolumeId:   volumeID,
-		Collection: collection,
-	})
-	if err == nil {
-		_, err = localDiskStream.Recv()
-	}
-	if err == nil || !strings.Contains(err.Error(), "already on local disk") {
-		t.Fatalf("VolumeTierMoveDatFromRemote local-disk error mismatch: %v", err)
-	}
-}

test/volume_server/grpc/vacuum_test.go

@@ -1,87 +0,0 @@
-package volume_server_grpc_test
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-)
-
-func TestVacuumVolumeCheckSuccessAndMissingVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(31)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	resp, err := grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: volumeID})
-	if err != nil {
-		t.Fatalf("VacuumVolumeCheck existing volume failed: %v", err)
-	}
-	if resp.GetGarbageRatio() < 0 || resp.GetGarbageRatio() > 1 {
-		t.Fatalf("unexpected garbage ratio: %f", resp.GetGarbageRatio())
-	}
-
-	_, err = grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: 99999})
-	if err == nil {
-		t.Fatalf("VacuumVolumeCheck should fail for missing volume")
-	}
-}
-
-func TestVacuumMaintenanceModeRejections(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
-	if err != nil {
-		t.Fatalf("GetState failed: %v", err)
-	}
-	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
-		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
-	})
-	if err != nil {
-		t.Fatalf("SetState maintenance=true failed: %v", err)
-	}
-
-	assertMaintenanceErr := func(name string, err error) {
-		t.Helper()
-		if err == nil {
-			t.Fatalf("%s should fail in maintenance mode", name)
-		}
-		if !strings.Contains(err.Error(), "maintenance mode") {
-			t.Fatalf("%s expected maintenance mode error, got: %v", name, err)
-		}
-	}
-
-	compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{VolumeId: 31})
-	if err == nil {
-		_, err = compactStream.Recv()
-	}
-	assertMaintenanceErr("VacuumVolumeCompact", err)
-
-	_, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 31})
-	assertMaintenanceErr("VacuumVolumeCommit", err)
-
-	_, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{VolumeId: 31})
-	assertMaintenanceErr("VacuumVolumeCleanup", err)
-}

test/volume_server/http/admin_test.go

@@ -1,174 +0,0 @@
-package volume_server_http_test
-
-import (
-	"encoding/json"
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
-)
-
-func TestAdminStatusAndHealthz(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	client := framework.NewHTTPClient()
-
-	statusReq, err := http.NewRequest(http.MethodGet, cluster.VolumeAdminURL()+"/status", nil)
-	if err != nil {
-		t.Fatalf("create status request: %v", err)
-	}
-	statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
-
-	statusResp := framework.DoRequest(t, client, statusReq)
-	statusBody := framework.ReadAllAndClose(t, statusResp)
-
-	if statusResp.StatusCode != http.StatusOK {
-		t.Fatalf("expected /status code 200, got %d, body: %s", statusResp.StatusCode, string(statusBody))
-	}
-	if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
-		t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
-	}
-	if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
-		t.Fatalf("expected echoed request id, got %q", got)
-	}
-
-	var payload map[string]interface{}
-	if err := json.Unmarshal(statusBody, &payload); err != nil {
-		t.Fatalf("decode status response: %v", err)
-	}
-	for _, field := range []string{"Version", "DiskStatuses", "Volumes"} {
-		if _, found := payload[field]; !found {
-			t.Fatalf("status payload missing field %q", field)
-		}
-	}
-
-	healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
-	healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
-	healthResp := framework.DoRequest(t, client, healthReq)
-	_ = framework.ReadAllAndClose(t, healthResp)
-	if healthResp.StatusCode != http.StatusOK {
-		t.Fatalf("expected /healthz code 200, got %d", healthResp.StatusCode)
-	}
-	if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
-		t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
-	}
-	if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
-		t.Fatalf("expected /healthz echoed request id, got %q", got)
-	}
-
-	uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
-	uiBody := framework.ReadAllAndClose(t, uiResp)
-	if uiResp.StatusCode != http.StatusOK {
-		t.Fatalf("expected /ui/index.html code 200, got %d, body: %s", uiResp.StatusCode, string(uiBody))
-	}
-	if !strings.Contains(strings.ToLower(string(uiBody)), "volume") {
-		t.Fatalf("ui page does not look like volume status page")
-	}
-}
-
-func TestOptionsMethodsByPort(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
-	client := framework.NewHTTPClient()
-
-	adminResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/"))
-	_ = framework.ReadAllAndClose(t, adminResp)
-	if adminResp.StatusCode != http.StatusOK {
-		t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode)
-	}
-	adminAllowed := adminResp.Header.Get("Access-Control-Allow-Methods")
-	for _, expected := range []string{"PUT", "POST", "GET", "DELETE", "OPTIONS"} {
-		if !strings.Contains(adminAllowed, expected) {
-			t.Fatalf("admin allow methods missing %q, got %q", expected, adminAllowed)
-		}
-	}
-	if adminResp.Header.Get("Access-Control-Allow-Headers") != "*" {
-		t.Fatalf("admin allow headers expected '*', got %q", adminResp.Header.Get("Access-Control-Allow-Headers"))
-	}
-
-	publicResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/"))
-	_ = framework.ReadAllAndClose(t, publicResp)
-	if publicResp.StatusCode != http.StatusOK {
-		t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode)
-	}
-	publicAllowed := publicResp.Header.Get("Access-Control-Allow-Methods")
-	if !strings.Contains(publicAllowed, "GET") || !strings.Contains(publicAllowed, "OPTIONS") {
-		t.Fatalf("public allow methods expected GET and OPTIONS, got %q", publicAllowed)
-	}
-	if strings.Contains(publicAllowed, "POST") {
-		t.Fatalf("public allow methods should not include POST, got %q", publicAllowed)
-	}
-	if publicResp.Header.Get("Access-Control-Allow-Headers") != "*" {
-		t.Fatalf("public allow headers expected '*', got %q", publicResp.Header.Get("Access-Control-Allow-Headers"))
-	}
-}
-
-func TestOptionsWithOriginIncludesCorsHeaders(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
-	client := framework.NewHTTPClient()
-
-	adminReq := mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/")
-	adminReq.Header.Set("Origin", "https://example.com")
-	adminResp := framework.DoRequest(t, client, adminReq)
-	_ = framework.ReadAllAndClose(t, adminResp)
-	if adminResp.StatusCode != http.StatusOK {
-		t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode)
-	}
-	if adminResp.Header.Get("Access-Control-Allow-Origin") != "*" {
-		t.Fatalf("admin OPTIONS expected Access-Control-Allow-Origin=*, got %q", adminResp.Header.Get("Access-Control-Allow-Origin"))
-	}
-	if adminResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
-		t.Fatalf("admin OPTIONS expected Access-Control-Allow-Credentials=true, got %q", adminResp.Header.Get("Access-Control-Allow-Credentials"))
-	}
-
-	publicReq := mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/")
-	publicReq.Header.Set("Origin", "https://example.com")
-	publicResp := framework.DoRequest(t, client, publicReq)
-	_ = framework.ReadAllAndClose(t, publicResp)
-	if publicResp.StatusCode != http.StatusOK {
-		t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode)
-	}
-	if publicResp.Header.Get("Access-Control-Allow-Origin") != "*" {
-		t.Fatalf("public OPTIONS expected Access-Control-Allow-Origin=*, got %q", publicResp.Header.Get("Access-Control-Allow-Origin"))
-	}
-	if publicResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
-		t.Fatalf("public OPTIONS expected Access-Control-Allow-Credentials=true, got %q", publicResp.Header.Get("Access-Control-Allow-Credentials"))
-	}
-}
-
-func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P3())
-	client := framework.NewHTTPClient()
-
-	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
-	body := framework.ReadAllAndClose(t, resp)
-	if resp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("expected /ui/index.html to be gated by auth under JWT profile (401), got %d body=%s", resp.StatusCode, string(body))
-	}
-}
-
-func mustNewRequest(t testing.TB, method, url string) *http.Request {
-	t.Helper()
-	req, err := http.NewRequest(method, url, nil)
-	if err != nil {
-		t.Fatalf("create request %s %s: %v", method, url, err)
-	}
-	return req
-}

test/volume_server/http/auth_test.go

@@ -1,419 +0,0 @@
-package volume_server_http_test
-
-import (
-	"bytes"
-	"net/http"
-	"testing"
-	"time"
-
-	jwt "github.com/golang-jwt/jwt/v5"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/security"
-)
-
-func TestJWTAuthForWriteAndRead(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(51)
-	const needleID = uint64(123456)
-	const cookie = uint32(0xABCDEF12)
-
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	payload := []byte("jwt-protected-content")
-	client := framework.NewHTTPClient()
-
-	unauthWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	unauthWriteResp := framework.DoRequest(t, client, unauthWrite)
-	_ = framework.ReadAllAndClose(t, unauthWriteResp)
-	if unauthWriteResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("unauthorized write expected 401, got %d", unauthWriteResp.StatusCode)
-	}
-
-	invalidWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	invalidWrite.Header.Set("Authorization", "Bearer invalid")
-	invalidWriteResp := framework.DoRequest(t, client, invalidWrite)
-	_ = framework.ReadAllAndClose(t, invalidWriteResp)
-	if invalidWriteResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("invalid write token expected 401, got %d", invalidWriteResp.StatusCode)
-	}
-
-	writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
-	authWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	authWrite.Header.Set("Authorization", "Bearer "+string(writeToken))
-	authWriteResp := framework.DoRequest(t, client, authWrite)
-	_ = framework.ReadAllAndClose(t, authWriteResp)
-	if authWriteResp.StatusCode != http.StatusCreated {
-		t.Fatalf("authorized write expected 201, got %d", authWriteResp.StatusCode)
-	}
-
-	unauthReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	unauthReadResp := framework.DoRequest(t, client, unauthReadReq)
-	_ = framework.ReadAllAndClose(t, unauthReadResp)
-	if unauthReadResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("unauthorized read expected 401, got %d", unauthReadResp.StatusCode)
-	}
-
-	readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
-	authReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	authReadReq.Header.Set("Authorization", "Bearer "+string(readToken))
-	authReadResp := framework.DoRequest(t, client, authReadReq)
-	authReadBody := framework.ReadAllAndClose(t, authReadResp)
-	if authReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("authorized read expected 200, got %d", authReadResp.StatusCode)
-	}
-	if string(authReadBody) != string(payload) {
-		t.Fatalf("authorized read content mismatch: got %q want %q", string(authReadBody), string(payload))
-	}
-}
-
-func TestJWTAuthRejectsFidMismatch(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(52)
-	const needleID = uint64(223344)
-	const cookie = uint32(0x10203040)
-	const otherNeedleID = uint64(223345)
-	const otherCookie = uint32(0x50607080)
-	const wrongCookie = uint32(0x10203041)
-
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	otherFid := framework.NewFileID(volumeID, otherNeedleID, otherCookie)
-	payload := []byte("jwt-fid-mismatch-content")
-	client := framework.NewHTTPClient()
-
-	writeTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFid)
-	mismatchedWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	mismatchedWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForOtherFid))
-	mismatchedWriteResp := framework.DoRequest(t, client, mismatchedWrite)
-	_ = framework.ReadAllAndClose(t, mismatchedWriteResp)
-	if mismatchedWriteResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("write with mismatched fid token expected 401, got %d", mismatchedWriteResp.StatusCode)
-	}
-
-	wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie)
-	writeTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, wrongCookieFid)
-	wrongCookieWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	wrongCookieWrite.Header.Set("Authorization", "Bearer "+string(writeTokenWrongCookie))
-	wrongCookieWriteResp := framework.DoRequest(t, client, wrongCookieWrite)
-	_ = framework.ReadAllAndClose(t, wrongCookieWriteResp)
-	if wrongCookieWriteResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("write with wrong-cookie fid token expected 401, got %d", wrongCookieWriteResp.StatusCode)
-	}
-
-	writeTokenForFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
-	validWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	validWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForFid))
-	validWriteResp := framework.DoRequest(t, client, validWrite)
-	_ = framework.ReadAllAndClose(t, validWriteResp)
-	if validWriteResp.StatusCode != http.StatusCreated {
-		t.Fatalf("authorized write expected 201, got %d", validWriteResp.StatusCode)
-	}
-
-	readTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFid)
-	mismatchedReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	mismatchedReadReq.Header.Set("Authorization", "Bearer "+string(readTokenForOtherFid))
-	mismatchedReadResp := framework.DoRequest(t, client, mismatchedReadReq)
-	_ = framework.ReadAllAndClose(t, mismatchedReadResp)
-	if mismatchedReadResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("read with mismatched fid token expected 401, got %d", mismatchedReadResp.StatusCode)
-	}
-
-	readTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, wrongCookieFid)
-	wrongCookieReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	wrongCookieReadReq.Header.Set("Authorization", "Bearer "+string(readTokenWrongCookie))
-	wrongCookieReadResp := framework.DoRequest(t, client, wrongCookieReadReq)
-	_ = framework.ReadAllAndClose(t, wrongCookieReadResp)
-	if wrongCookieReadResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("read with wrong-cookie fid token expected 401, got %d", wrongCookieReadResp.StatusCode)
-	}
-}
-
-func newUploadRequest(t testing.TB, url string, payload []byte) *http.Request {
-	t.Helper()
-	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload))
-	if err != nil {
-		t.Fatalf("create upload request %s: %v", url, err)
-	}
-	req.Header.Set("Content-Type", "application/octet-stream")
-	return req
-}
-
-func TestJWTAuthRejectsExpiredTokens(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(53)
-	const needleID = uint64(334455)
-	const cookie = uint32(0x22334455)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	payload := []byte("expired-token-content")
-	client := framework.NewHTTPClient()
-
-	expiredWriteToken := mustGenExpiredToken(t, []byte(profile.JWTSigningKey), fid)
-	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	writeReq.Header.Set("Authorization", "Bearer "+expiredWriteToken)
-	writeResp := framework.DoRequest(t, client, writeReq)
-	_ = framework.ReadAllAndClose(t, writeResp)
-	if writeResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("expired write token expected 401, got %d", writeResp.StatusCode)
-	}
-
-	// Seed data with a valid token so read auth path can be exercised against existing content.
-	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
-	validWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	validWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
-	validWriteResp := framework.DoRequest(t, client, validWriteReq)
-	_ = framework.ReadAllAndClose(t, validWriteResp)
-	if validWriteResp.StatusCode != http.StatusCreated {
-		t.Fatalf("valid write expected 201, got %d", validWriteResp.StatusCode)
-	}
-
-	expiredReadToken := mustGenExpiredToken(t, []byte(profile.JWTReadKey), fid)
-	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	readReq.Header.Set("Authorization", "Bearer "+expiredReadToken)
-	readResp := framework.DoRequest(t, client, readReq)
-	_ = framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("expired read token expected 401, got %d", readResp.StatusCode)
-	}
-}
-
-func TestJWTAuthViaQueryParamAndCookie(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(54)
-	const needleID = uint64(445566)
-	const cookie = uint32(0x31415926)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	payload := []byte("jwt-query-cookie-content")
-	client := framework.NewHTTPClient()
-
-	writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
-	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(writeToken), payload)
-	writeResp := framework.DoRequest(t, client, writeReq)
-	_ = framework.ReadAllAndClose(t, writeResp)
-	if writeResp.StatusCode != http.StatusCreated {
-		t.Fatalf("query-jwt write expected 201, got %d", writeResp.StatusCode)
-	}
-
-	readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
-	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(readToken)})
-	readResp := framework.DoRequest(t, client, readReq)
-	readBody := framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusOK {
-		t.Fatalf("cookie-jwt read expected 200, got %d", readResp.StatusCode)
-	}
-	if string(readBody) != string(payload) {
-		t.Fatalf("cookie-jwt read body mismatch: got %q want %q", string(readBody), string(payload))
-	}
-}
-
-func TestJWTTokenSourcePrecedenceQueryOverHeader(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(55)
-	const needleID = uint64(556677)
-	const cookie = uint32(0x99887766)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1)
-	payload := []byte("jwt-precedence-content")
-	client := framework.NewHTTPClient()
-
-	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
-	invalidWriteQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID)
-	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidWriteQueryToken), payload)
-	writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
-	writeResp := framework.DoRequest(t, client, writeReq)
-	_ = framework.ReadAllAndClose(t, writeResp)
-	if writeResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("query token should take precedence over header token for write, expected 401 got %d", writeResp.StatusCode)
-	}
-
-	// Seed data with valid write token, then exercise read precedence.
-	seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
-	seedWriteResp := framework.DoRequest(t, client, seedWriteReq)
-	_ = framework.ReadAllAndClose(t, seedWriteResp)
-	if seedWriteResp.StatusCode != http.StatusCreated {
-		t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode)
-	}
-
-	validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
-	invalidReadQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
-	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidReadQueryToken))
-	readReq.Header.Set("Authorization", "Bearer "+string(validReadToken))
-	readResp := framework.DoRequest(t, client, readReq)
-	_ = framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("query token should take precedence over header token for read, expected 401 got %d", readResp.StatusCode)
-	}
-}
-
-func TestJWTTokenSourcePrecedenceHeaderOverCookie(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(56)
-	const needleID = uint64(667788)
-	const cookie = uint32(0x11229988)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1)
-	payload := []byte("jwt-precedence-header-cookie")
-	client := framework.NewHTTPClient()
-
-	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
-	invalidCookieWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID)
-	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
-	writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieWriteToken)})
-	writeResp := framework.DoRequest(t, client, writeReq)
-	_ = framework.ReadAllAndClose(t, writeResp)
-	if writeResp.StatusCode != http.StatusCreated {
-		t.Fatalf("header token should take precedence over cookie token for write, expected 201 got %d", writeResp.StatusCode)
-	}
-
-	validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
-	invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
-	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	readReq.Header.Set("Authorization", "Bearer "+string(validReadToken))
-	readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)})
-	readResp := framework.DoRequest(t, client, readReq)
-	readBody := framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusOK {
-		t.Fatalf("header token should take precedence over cookie token for read, expected 200 got %d", readResp.StatusCode)
-	}
-	if string(readBody) != string(payload) {
-		t.Fatalf("header-over-cookie read body mismatch: got %q want %q", string(readBody), string(payload))
-	}
-}
-
-func TestJWTTokenSourcePrecedenceQueryOverCookie(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(57)
-	const needleID = uint64(778899)
-	const cookie = uint32(0x88776655)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1)
-	payload := []byte("jwt-precedence-query-cookie")
-	client := framework.NewHTTPClient()
-
-	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
-	invalidQueryWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID)
-	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryWriteToken), payload)
-	writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validWriteToken)})
-	writeResp := framework.DoRequest(t, client, writeReq)
-	_ = framework.ReadAllAndClose(t, writeResp)
-	if writeResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("query token should take precedence over cookie token for write, expected 401 got %d", writeResp.StatusCode)
-	}
-
-	// Seed data with valid write token so read precedence can be exercised.
-	seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
-	seedWriteResp := framework.DoRequest(t, client, seedWriteReq)
-	_ = framework.ReadAllAndClose(t, seedWriteResp)
-	if seedWriteResp.StatusCode != http.StatusCreated {
-		t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode)
-	}
-
-	validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
-	invalidQueryReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
-	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryReadToken))
-	readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validReadToken)})
-	readResp := framework.DoRequest(t, client, readReq)
-	_ = framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("query token should take precedence over cookie token for read, expected 401 got %d", readResp.StatusCode)
-	}
-
-	// Validate positive path: valid query token should succeed even if cookie token is invalid.
-	validQueryReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(validReadToken))
-	invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
-	validQueryReadReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)})
-	validQueryReadResp := framework.DoRequest(t, client, validQueryReadReq)
-	validQueryReadBody := framework.ReadAllAndClose(t, validQueryReadResp)
-	if validQueryReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("valid query token should succeed over invalid cookie token, expected 200 got %d", validQueryReadResp.StatusCode)
-	}
-	if string(validQueryReadBody) != string(payload) {
-		t.Fatalf("query-over-cookie read body mismatch: got %q want %q", string(validQueryReadBody), string(payload))
-	}
-}
-
-func mustGenExpiredToken(t testing.TB, key []byte, fid string) string {
-	t.Helper()
-	claims := security.SeaweedFileIdClaims{
-		Fid: fid,
-		RegisteredClaims: jwt.RegisteredClaims{
-			ExpiresAt: jwt.NewNumericDate(time.Now().Add(-1 * time.Minute)),
-		},
-	}
-	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
-	signed, err := token.SignedString(key)
-	if err != nil {
-		t.Fatalf("sign expired token: %v", err)
-	}
-	return signed
-}

test/volume_server/http/chunk_manifest_test.go

@@ -1,232 +0,0 @@
-package volume_server_http_test
-
-import (
-	"bytes"
-	"encoding/json"
-	"net/http"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/operation"
-)
-
-func TestChunkManifestExpansionAndBypass(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(102)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-
-	chunkFID := framework.NewFileID(volumeID, 772005, 0x5E6F7081)
-	chunkPayload := []byte("chunk-manifest-expanded-content")
-	chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload)
-	_ = framework.ReadAllAndClose(t, chunkUploadResp)
-	if chunkUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode)
-	}
-
-	manifest := &operation.ChunkManifest{
-		Name: "manifest.bin",
-		Mime: "application/octet-stream",
-		Size: int64(len(chunkPayload)),
-		Chunks: []*operation.ChunkInfo{
-			{
-				Fid:    chunkFID,
-				Offset: 0,
-				Size:   int64(len(chunkPayload)),
-			},
-		},
-	}
-	manifestBytes, err := json.Marshal(manifest)
-	if err != nil {
-		t.Fatalf("marshal chunk manifest: %v", err)
-	}
-
-	manifestFID := framework.NewFileID(volumeID, 772006, 0x6F708192)
-	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
-	if err != nil {
-		t.Fatalf("create manifest upload request: %v", err)
-	}
-	manifestUploadReq.Header.Set("Content-Type", "application/json")
-	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
-	_ = framework.ReadAllAndClose(t, manifestUploadResp)
-	if manifestUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
-	}
-
-	expandedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID)
-	expandedReadBody := framework.ReadAllAndClose(t, expandedReadResp)
-	if expandedReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("manifest expanded read expected 200, got %d", expandedReadResp.StatusCode)
-	}
-	if string(expandedReadBody) != string(chunkPayload) {
-		t.Fatalf("manifest expanded read mismatch: got %q want %q", string(expandedReadBody), string(chunkPayload))
-	}
-	if expandedReadResp.Header.Get("X-File-Store") != "chunked" {
-		t.Fatalf("manifest expanded read expected X-File-Store=chunked, got %q", expandedReadResp.Header.Get("X-File-Store"))
-	}
-
-	bypassReadResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false"))
-	bypassReadBody := framework.ReadAllAndClose(t, bypassReadResp)
-	if bypassReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("manifest bypass read expected 200, got %d", bypassReadResp.StatusCode)
-	}
-	if bypassReadResp.Header.Get("X-File-Store") != "" {
-		t.Fatalf("manifest bypass read expected empty X-File-Store header, got %q", bypassReadResp.Header.Get("X-File-Store"))
-	}
-
-	var gotManifest operation.ChunkManifest
-	if err = json.Unmarshal(bypassReadBody, &gotManifest); err != nil {
-		t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(bypassReadBody))
-	}
-	if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != chunkFID {
-		t.Fatalf("manifest bypass read payload mismatch: %+v", gotManifest)
-	}
-}
-
-func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(104)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-
-	chunkFID := framework.NewFileID(volumeID, 772008, 0x8192A3B4)
-	chunkPayload := []byte("chunk-manifest-delete-content")
-	chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload)
-	_ = framework.ReadAllAndClose(t, chunkUploadResp)
-	if chunkUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode)
-	}
-
-	manifest := &operation.ChunkManifest{
-		Name: "manifest-delete.bin",
-		Mime: "application/octet-stream",
-		Size: int64(len(chunkPayload)),
-		Chunks: []*operation.ChunkInfo{
-			{
-				Fid:    chunkFID,
-				Offset: 0,
-				Size:   int64(len(chunkPayload)),
-			},
-		},
-	}
-	manifestBytes, err := json.Marshal(manifest)
-	if err != nil {
-		t.Fatalf("marshal chunk manifest: %v", err)
-	}
-
-	manifestFID := framework.NewFileID(volumeID, 772009, 0x92A3B4C5)
-	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
-	if err != nil {
-		t.Fatalf("create manifest upload request: %v", err)
-	}
-	manifestUploadReq.Header.Set("Content-Type", "application/json")
-	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
-	_ = framework.ReadAllAndClose(t, manifestUploadResp)
-	if manifestUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
-	}
-
-	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID))
-	deleteBody := framework.ReadAllAndClose(t, deleteResp)
-	if deleteResp.StatusCode != http.StatusAccepted {
-		t.Fatalf("manifest delete expected 202, got %d", deleteResp.StatusCode)
-	}
-	var deleteResult map[string]int64
-	if err = json.Unmarshal(deleteBody, &deleteResult); err != nil {
-		t.Fatalf("decode manifest delete response: %v body=%q", err, string(deleteBody))
-	}
-	if deleteResult["size"] != int64(len(chunkPayload)) {
-		t.Fatalf("manifest delete expected size=%d, got %d", len(chunkPayload), deleteResult["size"])
-	}
-
-	manifestReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID)
-	_ = framework.ReadAllAndClose(t, manifestReadAfterDelete)
-	if manifestReadAfterDelete.StatusCode != http.StatusNotFound {
-		t.Fatalf("manifest read after delete expected 404, got %d", manifestReadAfterDelete.StatusCode)
-	}
-
-	chunkReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID)
-	_ = framework.ReadAllAndClose(t, chunkReadAfterDelete)
-	if chunkReadAfterDelete.StatusCode != http.StatusNotFound {
-		t.Fatalf("chunk read after manifest delete expected 404, got %d", chunkReadAfterDelete.StatusCode)
-	}
-}
-
-func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(105)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	manifest := &operation.ChunkManifest{
-		Name: "manifest-delete-failure.bin",
-		Mime: "application/octet-stream",
-		Size: 1,
-		Chunks: []*operation.ChunkInfo{
-			{
-				Fid:    "not-a-valid-fid",
-				Offset: 0,
-				Size:   1,
-			},
-		},
-	}
-	manifestBytes, err := json.Marshal(manifest)
-	if err != nil {
-		t.Fatalf("marshal chunk manifest: %v", err)
-	}
-
-	manifestFID := framework.NewFileID(volumeID, 772010, 0xA3B4C5D6)
-	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
-	if err != nil {
-		t.Fatalf("create manifest upload request: %v", err)
-	}
-	manifestUploadReq.Header.Set("Content-Type", "application/json")
-	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
-	_ = framework.ReadAllAndClose(t, manifestUploadResp)
-	if manifestUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
-	}
-
-	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID))
-	deleteBody := framework.ReadAllAndClose(t, deleteResp)
-	if deleteResp.StatusCode != http.StatusInternalServerError {
-		t.Fatalf("manifest delete with invalid child fid expected 500, got %d body=%q", deleteResp.StatusCode, string(deleteBody))
-	}
-
-	manifestBypassRead := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false"))
-	manifestBypassBody := framework.ReadAllAndClose(t, manifestBypassRead)
-	if manifestBypassRead.StatusCode != http.StatusOK {
-		t.Fatalf("manifest bypass read after failed delete expected 200, got %d", manifestBypassRead.StatusCode)
-	}
-	var gotManifest operation.ChunkManifest
-	if err = json.Unmarshal(manifestBypassBody, &gotManifest); err != nil {
-		t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(manifestBypassBody))
-	}
-	if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != "not-a-valid-fid" {
-		t.Fatalf("manifest payload mismatch after failed delete: %+v", gotManifest)
-	}
-}

test/volume_server/http/compressed_read_test.go

@@ -1,97 +0,0 @@
-package volume_server_http_test
-
-import (
-	"bytes"
-	"compress/gzip"
-	"io"
-	"net/http"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func gzipData(t testing.TB, data []byte) []byte {
-	t.Helper()
-	var buf bytes.Buffer
-	zw := gzip.NewWriter(&buf)
-	if _, err := zw.Write(data); err != nil {
-		t.Fatalf("gzip write: %v", err)
-	}
-	if err := zw.Close(); err != nil {
-		t.Fatalf("gzip close: %v", err)
-	}
-	return buf.Bytes()
-}
-
-func gunzipData(t testing.TB, data []byte) []byte {
-	t.Helper()
-	zr, err := gzip.NewReader(bytes.NewReader(data))
-	if err != nil {
-		t.Fatalf("gunzip new reader: %v", err)
-	}
-	defer zr.Close()
-	out, err := io.ReadAll(zr)
-	if err != nil {
-		t.Fatalf("gunzip read: %v", err)
-	}
-	return out
-}
-
-func TestCompressedReadAcceptEncodingMatrix(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(103)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 772007, 0x708192A3)
-	plainPayload := []byte("compressed-read-accept-encoding-matrix-content-compressed-read-accept-encoding-matrix-content")
-	compressedPayload := gzipData(t, plainPayload)
-
-	uploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader(compressedPayload))
-	if err != nil {
-		t.Fatalf("create compressed upload request: %v", err)
-	}
-	uploadReq.Header.Set("Content-Type", "text/plain")
-	uploadReq.Header.Set("Content-Encoding", "gzip")
-	uploadResp := framework.DoRequest(t, client, uploadReq)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("compressed upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	gzipReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	gzipReadReq.Header.Set("Accept-Encoding", "gzip")
-	gzipReadResp := framework.DoRequest(t, client, gzipReadReq)
-	gzipReadBody := framework.ReadAllAndClose(t, gzipReadResp)
-	if gzipReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("gzip-accepted read expected 200, got %d", gzipReadResp.StatusCode)
-	}
-	if gzipReadResp.Header.Get("Content-Encoding") != "gzip" {
-		t.Fatalf("gzip-accepted read expected Content-Encoding=gzip, got %q", gzipReadResp.Header.Get("Content-Encoding"))
-	}
-	if string(gunzipData(t, gzipReadBody)) != string(plainPayload) {
-		t.Fatalf("gzip-accepted read body mismatch after gunzip")
-	}
-
-	identityReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	identityReadReq.Header.Set("Accept-Encoding", "identity")
-	identityReadResp := framework.DoRequest(t, client, identityReadReq)
-	identityReadBody := framework.ReadAllAndClose(t, identityReadResp)
-	if identityReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("identity read expected 200, got %d", identityReadResp.StatusCode)
-	}
-	if identityReadResp.Header.Get("Content-Encoding") != "" {
-		t.Fatalf("identity read expected no Content-Encoding header, got %q", identityReadResp.Header.Get("Content-Encoding"))
-	}
-	if string(identityReadBody) != string(plainPayload) {
-		t.Fatalf("identity read body mismatch: got %q want %q", string(identityReadBody), string(plainPayload))
-	}
-}

test/volume_server/http/headers_static_test.go

@@ -1,102 +0,0 @@
-package volume_server_http_test
-
-import (
-	"fmt"
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(96)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fullFileID := framework.NewFileID(volumeID, 661122, 0x55667788)
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, []byte("passthrough-header-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	parts := strings.SplitN(fullFileID, ",", 2)
-	if len(parts) != 2 {
-		t.Fatalf("unexpected file id format: %q", fullFileID)
-	}
-	fidOnly := parts[1]
-
-	url := fmt.Sprintf("%s/%d/%s/%s?response-content-type=text/plain&response-cache-control=no-store&dl=true",
-		clusterHarness.VolumeAdminURL(),
-		volumeID,
-		fidOnly,
-		"report.txt",
-	)
-	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url))
-	_ = framework.ReadAllAndClose(t, resp)
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("passthrough read expected 200, got %d", resp.StatusCode)
-	}
-	if resp.Header.Get("Content-Type") != "text/plain" {
-		t.Fatalf("response-content-type override mismatch: %q", resp.Header.Get("Content-Type"))
-	}
-	if resp.Header.Get("Cache-Control") != "no-store" {
-		t.Fatalf("response-cache-control override mismatch: %q", resp.Header.Get("Cache-Control"))
-	}
-	contentDisposition := resp.Header.Get("Content-Disposition")
-	if !strings.Contains(contentDisposition, "attachment") || !strings.Contains(contentDisposition, "report.txt") {
-		t.Fatalf("download disposition header mismatch: %q", contentDisposition)
-	}
-}
-
-func TestStaticAssetEndpoints(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	client := framework.NewHTTPClient()
-
-	faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/favicon.ico"))
-	_ = framework.ReadAllAndClose(t, faviconResp)
-	if faviconResp.StatusCode != http.StatusOK {
-		t.Fatalf("/favicon.ico expected 200, got %d", faviconResp.StatusCode)
-	}
-
-	staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/seaweedfsstatic/seaweed50x50.png"))
-	_ = framework.ReadAllAndClose(t, staticResp)
-	if staticResp.StatusCode != http.StatusOK {
-		t.Fatalf("/seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode)
-	}
-}
-
-func TestStaticAssetEndpointsOnPublicPort(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	client := framework.NewHTTPClient()
-
-	faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/favicon.ico"))
-	_ = framework.ReadAllAndClose(t, faviconResp)
-	if faviconResp.StatusCode != http.StatusOK {
-		t.Fatalf("public /favicon.ico expected 200, got %d", faviconResp.StatusCode)
-	}
-
-	staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/seaweedfsstatic/seaweed50x50.png"))
-	_ = framework.ReadAllAndClose(t, staticResp)
-	if staticResp.StatusCode != http.StatusOK {
-		t.Fatalf("public /seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode)
-	}
-}

test/volume_server/http/image_transform_test.go

@@ -1,92 +0,0 @@
-package volume_server_http_test
-
-import (
-	"bytes"
-	"fmt"
-	"image"
-	"image/color"
-	"image/png"
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func makePNGFixture(t testing.TB, width, height int) []byte {
-	t.Helper()
-
-	img := image.NewRGBA(image.Rect(0, 0, width, height))
-	for y := 0; y < height; y++ {
-		for x := 0; x < width; x++ {
-			img.Set(x, y, color.RGBA{R: uint8(x * 20), G: uint8(y * 20), B: 200, A: 255})
-		}
-	}
-
-	var buf bytes.Buffer
-	if err := png.Encode(&buf, img); err != nil {
-		t.Fatalf("encode png fixture: %v", err)
-	}
-	return buf.Bytes()
-}
-
-func decodeImageConfig(t testing.TB, data []byte) image.Config {
-	t.Helper()
-	cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
-	if err != nil {
-		t.Fatalf("decode image config: %v", err)
-	}
-	return cfg
-}
-
-func TestImageResizeAndCropReadVariants(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(101)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fullFileID := framework.NewFileID(volumeID, 772004, 0x4D5E6F70)
-	uploadReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fullFileID, makePNGFixture(t, 6, 4))
-	uploadReq.Header.Set("Content-Type", "image/png")
-	uploadResp := framework.DoRequest(t, client, uploadReq)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("image upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	parts := strings.SplitN(fullFileID, ",", 2)
-	if len(parts) != 2 {
-		t.Fatalf("unexpected file id format: %q", fullFileID)
-	}
-	fidOnly := parts[1]
-
-	resizeURL := fmt.Sprintf("%s/%d/%s/%s?width=2&height=1", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png")
-	resizeResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, resizeURL))
-	resizeBody := framework.ReadAllAndClose(t, resizeResp)
-	if resizeResp.StatusCode != http.StatusOK {
-		t.Fatalf("image resize read expected 200, got %d", resizeResp.StatusCode)
-	}
-	resizeCfg := decodeImageConfig(t, resizeBody)
-	if resizeCfg.Width > 2 || resizeCfg.Height > 1 {
-		t.Fatalf("image resize expected dimensions <= 2x1, got %dx%d", resizeCfg.Width, resizeCfg.Height)
-	}
-
-	cropURL := fmt.Sprintf("%s/%d/%s/%s?crop_x1=1&crop_y1=1&crop_x2=4&crop_y2=3", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png")
-	cropResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cropURL))
-	cropBody := framework.ReadAllAndClose(t, cropResp)
-	if cropResp.StatusCode != http.StatusOK {
-		t.Fatalf("image crop read expected 200, got %d", cropResp.StatusCode)
-	}
-	cropCfg := decodeImageConfig(t, cropBody)
-	if cropCfg.Width != 3 || cropCfg.Height != 2 {
-		t.Fatalf("image crop expected 3x2, got %dx%d", cropCfg.Width, cropCfg.Height)
-	}
-}

test/volume_server/http/public_cors_methods_test.go

@@ -1,287 +0,0 @@
-package volume_server_http_test
-
-import (
-	"bytes"
-	"net/http"
-	"strconv"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestPublicPortReadOnlyMethodBehavior(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(81)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 123321, 0x01020304)
-	originalData := []byte("public-port-original")
-	replacementData := []byte("public-port-replacement")
-	client := framework.NewHTTPClient()
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, originalData)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	publicReadResp := framework.ReadBytes(t, client, clusterHarness.VolumePublicURL(), fid)
-	publicReadBody := framework.ReadAllAndClose(t, publicReadResp)
-	if publicReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("public GET expected 200, got %d", publicReadResp.StatusCode)
-	}
-	if string(publicReadBody) != string(originalData) {
-		t.Fatalf("public GET body mismatch: got %q want %q", string(publicReadBody), string(originalData))
-	}
-
-	publicPostReq := newUploadRequest(t, clusterHarness.VolumePublicURL()+"/"+fid, replacementData)
-	publicPostResp := framework.DoRequest(t, client, publicPostReq)
-	_ = framework.ReadAllAndClose(t, publicPostResp)
-	if publicPostResp.StatusCode != http.StatusOK {
-		t.Fatalf("public POST expected passthrough 200, got %d", publicPostResp.StatusCode)
-	}
-
-	publicDeleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumePublicURL()+"/"+fid))
-	_ = framework.ReadAllAndClose(t, publicDeleteResp)
-	if publicDeleteResp.StatusCode != http.StatusOK {
-		t.Fatalf("public DELETE expected passthrough 200, got %d", publicDeleteResp.StatusCode)
-	}
-
-	adminReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
-	adminReadBody := framework.ReadAllAndClose(t, adminReadResp)
-	if adminReadResp.StatusCode != http.StatusOK {
-		t.Fatalf("admin GET after public POST/DELETE expected 200, got %d", adminReadResp.StatusCode)
-	}
-	if string(adminReadBody) != string(originalData) {
-		t.Fatalf("public port should not mutate data: got %q want %q", string(adminReadBody), string(originalData))
-	}
-}
-
-func TestCorsAndUnsupportedMethodBehavior(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(82)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 789789, 0x0A0B0C0D)
-	client := framework.NewHTTPClient()
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("cors-check"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	adminOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	adminOriginReq.Header.Set("Origin", "https://example.com")
-	adminOriginResp := framework.DoRequest(t, client, adminOriginReq)
-	_ = framework.ReadAllAndClose(t, adminOriginResp)
-	if adminOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" {
-		t.Fatalf("admin GET origin header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Origin"))
-	}
-	if adminOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
-		t.Fatalf("admin GET credentials header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Credentials"))
-	}
-
-	publicOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/"+fid)
-	publicOriginReq.Header.Set("Origin", "https://example.com")
-	publicOriginResp := framework.DoRequest(t, client, publicOriginReq)
-	_ = framework.ReadAllAndClose(t, publicOriginResp)
-	if publicOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" {
-		t.Fatalf("public GET origin header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Origin"))
-	}
-	if publicOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
-		t.Fatalf("public GET credentials header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Credentials"))
-	}
-
-	adminPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader([]byte("patch")))
-	if err != nil {
-		t.Fatalf("create admin PATCH request: %v", err)
-	}
-	adminPatchResp := framework.DoRequest(t, client, adminPatchReq)
-	_ = framework.ReadAllAndClose(t, adminPatchResp)
-	if adminPatchResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("admin PATCH expected 400, got %d", adminPatchResp.StatusCode)
-	}
-
-	publicPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumePublicURL()+"/"+fid, bytes.NewReader([]byte("patch")))
-	if err != nil {
-		t.Fatalf("create public PATCH request: %v", err)
-	}
-	publicPatchResp := framework.DoRequest(t, client, publicPatchReq)
-	_ = framework.ReadAllAndClose(t, publicPatchResp)
-	if publicPatchResp.StatusCode != http.StatusOK {
-		t.Fatalf("public PATCH expected passthrough 200, got %d", publicPatchResp.StatusCode)
-	}
-}
-
-func TestUnsupportedMethodTraceParity(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(83)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 123999, 0x01010101)
-	client := framework.NewHTTPClient()
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("trace-method-check"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	adminTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumeAdminURL()+"/"+fid)
-	adminTraceResp := framework.DoRequest(t, client, adminTraceReq)
-	_ = framework.ReadAllAndClose(t, adminTraceResp)
-	if adminTraceResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("admin TRACE expected 400, got %d", adminTraceResp.StatusCode)
-	}
-
-	publicTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumePublicURL()+"/"+fid)
-	publicTraceResp := framework.DoRequest(t, client, publicTraceReq)
-	_ = framework.ReadAllAndClose(t, publicTraceResp)
-	if publicTraceResp.StatusCode != http.StatusOK {
-		t.Fatalf("public TRACE expected passthrough 200, got %d", publicTraceResp.StatusCode)
-	}
-}
-
-func TestUnsupportedMethodPropfindParity(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(84)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 124000, 0x02020202)
-	client := framework.NewHTTPClient()
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("propfind-method-check"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	adminReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumeAdminURL()+"/"+fid)
-	adminResp := framework.DoRequest(t, client, adminReq)
-	_ = framework.ReadAllAndClose(t, adminResp)
-	if adminResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("admin PROPFIND expected 400, got %d", adminResp.StatusCode)
-	}
-
-	publicReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumePublicURL()+"/"+fid)
-	publicResp := framework.DoRequest(t, client, publicReq)
-	_ = framework.ReadAllAndClose(t, publicResp)
-	if publicResp.StatusCode != http.StatusOK {
-		t.Fatalf("public PROPFIND expected passthrough 200, got %d", publicResp.StatusCode)
-	}
-
-	verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
-	verifyBody := framework.ReadAllAndClose(t, verifyResp)
-	if verifyResp.StatusCode != http.StatusOK {
-		t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
-	}
-	if string(verifyBody) != "propfind-method-check" {
-		t.Fatalf("PROPFIND should not mutate data, got %q", string(verifyBody))
-	}
-}
-
-func TestUnsupportedMethodConnectParity(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(85)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 124001, 0x03030303)
-	client := framework.NewHTTPClient()
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("connect-method-check"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	adminReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumeAdminURL()+"/"+fid)
-	adminResp := framework.DoRequest(t, client, adminReq)
-	_ = framework.ReadAllAndClose(t, adminResp)
-	if adminResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("admin CONNECT expected 400, got %d", adminResp.StatusCode)
-	}
-
-	publicReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumePublicURL()+"/"+fid)
-	publicResp := framework.DoRequest(t, client, publicReq)
-	_ = framework.ReadAllAndClose(t, publicResp)
-	if publicResp.StatusCode != http.StatusOK {
-		t.Fatalf("public CONNECT expected passthrough 200, got %d", publicResp.StatusCode)
-	}
-
-	verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
-	verifyBody := framework.ReadAllAndClose(t, verifyResp)
-	if verifyResp.StatusCode != http.StatusOK {
-		t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
-	}
-	if string(verifyBody) != "connect-method-check" {
-		t.Fatalf("CONNECT should not mutate data, got %q", string(verifyBody))
-	}
-}
-
-func TestPublicPortHeadReadParity(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(86)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 124002, 0x04040404)
-	payload := []byte("public-head-parity-content")
-	client := framework.NewHTTPClient()
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid))
-	headBody := framework.ReadAllAndClose(t, headResp)
-	if headResp.StatusCode != http.StatusOK {
-		t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode)
-	}
-	if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
-		t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload))
-	}
-	if len(headBody) != 0 {
-		t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
-	}
-}

test/volume_server/http/range_variants_test.go

@@ -1,82 +0,0 @@
-package volume_server_http_test
-
-import (
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestMultiRangeReadReturnsMultipartPayload(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(97)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 771999, 0x0A1B2C3D)
-	payload := []byte("0123456789abcdef")
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	multiRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	multiRangeReq.Header.Set("Range", "bytes=0-1,4-5")
-	multiRangeResp := framework.DoRequest(t, client, multiRangeReq)
-	multiRangeBody := framework.ReadAllAndClose(t, multiRangeResp)
-	if multiRangeResp.StatusCode != http.StatusPartialContent {
-		t.Fatalf("multi-range expected 206, got %d", multiRangeResp.StatusCode)
-	}
-	if !strings.Contains(multiRangeResp.Header.Get("Content-Type"), "multipart/byteranges") {
-		t.Fatalf("multi-range content-type mismatch: %q", multiRangeResp.Header.Get("Content-Type"))
-	}
-
-	bodyText := string(multiRangeBody)
-	if !strings.Contains(bodyText, "01") || !strings.Contains(bodyText, "45") {
-		t.Fatalf("multi-range body missing expected segments: %q", bodyText)
-	}
-}
-
-func TestOversizedCombinedRangesAreIgnored(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(100)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 772003, 0x3C4D5E6F)
-	payload := []byte("0123456789abcdef")
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	// Range bytes sum is 22 (> payload size 16), which exercises the oversized-range guard path.
-	oversizedRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	oversizedRangeReq.Header.Set("Range", "bytes=0-10,5-15")
-	oversizedRangeResp := framework.DoRequest(t, client, oversizedRangeReq)
-	oversizedRangeBody := framework.ReadAllAndClose(t, oversizedRangeResp)
-	if oversizedRangeResp.StatusCode != http.StatusOK {
-		t.Fatalf("oversized combined range expected 200, got %d", oversizedRangeResp.StatusCode)
-	}
-	if len(oversizedRangeBody) != 0 {
-		t.Fatalf("oversized combined range expected empty body, got %d bytes", len(oversizedRangeBody))
-	}
-}

test/volume_server/http/read_deleted_test.go

@@ -1,54 +0,0 @@
-package volume_server_http_test
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(94)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 551234, 0xCAFE1234)
-	payload := []byte("read-deleted-needle-payload")
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid))
-	_ = framework.ReadAllAndClose(t, deleteResp)
-	if deleteResp.StatusCode != http.StatusAccepted {
-		t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
-	}
-
-	normalRead := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
-	_ = framework.ReadAllAndClose(t, normalRead)
-	if normalRead.StatusCode != http.StatusNotFound {
-		t.Fatalf("normal read after delete expected 404, got %d", normalRead.StatusCode)
-	}
-
-	readDeletedReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")
-	readDeletedResp := framework.DoRequest(t, client, readDeletedReq)
-	readDeletedBody := framework.ReadAllAndClose(t, readDeletedResp)
-	if readDeletedResp.StatusCode != http.StatusOK {
-		t.Fatalf("read with readDeleted=true expected 200, got %d", readDeletedResp.StatusCode)
-	}
-	if string(readDeletedBody) != string(payload) {
-		t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload))
-	}
-}

test/volume_server/http/read_mode_proxy_redirect_test.go

@@ -1,319 +0,0 @@
-package volume_server_http_test
-
-import (
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestReadModeProxyMissingLocalVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P1()
-	profile.ReadMode = "proxy"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-
-	const volumeID = uint32(101)
-	framework.AllocateVolume(t, grpc0, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 120001, 0x0102ABCD)
-	payload := []byte("proxy-read-mode-forwarded-content")
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
-	var finalBody []byte
-	if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
-		finalBody = framework.ReadAllAndClose(t, resp)
-	}) {
-		t.Fatalf("proxy read mode did not return 200 from non-owning volume server within deadline")
-	}
-	if string(finalBody) != string(payload) {
-		t.Fatalf("proxy read mode body mismatch: got %q want %q", string(finalBody), string(payload))
-	}
-}
-
-func TestReadModeRedirectMissingLocalVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P1()
-	profile.ReadMode = "redirect"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-
-	const volumeID = uint32(102)
-	framework.AllocateVolume(t, grpc0, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 120002, 0x0102DCBA)
-	payload := []byte("redirect-read-mode-content")
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	noRedirectClient := &http.Client{
-		Timeout: 10 * time.Second,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-	}
-
-	readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
-	var redirectLocation string
-	if !waitForHTTPStatus(t, noRedirectClient, readURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) {
-		redirectLocation = resp.Header.Get("Location")
-		_ = framework.ReadAllAndClose(t, resp)
-	}) {
-		t.Fatalf("redirect read mode did not return 301 from non-owning volume server within deadline")
-	}
-	if redirectLocation == "" {
-		t.Fatalf("redirect response missing Location header")
-	}
-	if !strings.Contains(redirectLocation, "proxied=true") {
-		t.Fatalf("redirect Location should include proxied=true, got %q", redirectLocation)
-	}
-
-	followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, redirectLocation))
-	followBody := framework.ReadAllAndClose(t, followResp)
-	if followResp.StatusCode != http.StatusOK {
-		t.Fatalf("following redirect expected 200, got %d", followResp.StatusCode)
-	}
-	if string(followBody) != string(payload) {
-		t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload))
-	}
-}
-
-func TestReadModeLocalMissingLocalVolumeReturnsNotFound(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P1()
-	profile.ReadMode = "local"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-
-	const volumeID = uint32(103)
-	framework.AllocateVolume(t, grpc0, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 120003, 0x0102BEEF)
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, []byte("local-read-mode-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(1), fid)
-	_ = framework.ReadAllAndClose(t, readResp)
-	if readResp.StatusCode != http.StatusNotFound {
-		t.Fatalf("local read mode expected 404 on non-owning server, got %d", readResp.StatusCode)
-	}
-}
-
-func TestReadDeletedProxyModeOnMissingLocalVolume(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P1()
-	profile.ReadMode = "proxy"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-
-	const volumeID = uint32(104)
-	framework.AllocateVolume(t, grpc0, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 120004, 0x0102CAFE)
-	payload := []byte("proxy-readDeleted-missing-local-content")
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid))
-	_ = framework.ReadAllAndClose(t, deleteResp)
-	if deleteResp.StatusCode != http.StatusAccepted {
-		t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
-	}
-
-	readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true"
-	var proxiedBody []byte
-	if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
-		proxiedBody = framework.ReadAllAndClose(t, resp)
-	}) {
-		t.Fatalf("proxy readDeleted path did not return 200 from non-owning volume server within deadline")
-	}
-	if string(proxiedBody) != string(payload) {
-		t.Fatalf("proxy readDeleted body mismatch: got %q want %q", string(proxiedBody), string(payload))
-	}
-}
-
-func TestReadDeletedRedirectModeDropsQueryParameterParity(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P1()
-	profile.ReadMode = "redirect"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-
-	const volumeID = uint32(105)
-	framework.AllocateVolume(t, grpc0, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 120005, 0x0102FACE)
-	payload := []byte("redirect-readDeleted-query-drop-parity")
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid))
-	_ = framework.ReadAllAndClose(t, deleteResp)
-	if deleteResp.StatusCode != http.StatusAccepted {
-		t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
-	}
-
-	noRedirectClient := &http.Client{
-		Timeout: 10 * time.Second,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-	}
-
-	redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true"
-	var location string
-	if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) {
-		location = resp.Header.Get("Location")
-		_ = framework.ReadAllAndClose(t, resp)
-	}) {
-		t.Fatalf("redirect readDeleted path did not return 301 from non-owning volume server within deadline")
-	}
-	if location == "" {
-		t.Fatalf("redirect readDeleted response missing Location header")
-	}
-	if !strings.Contains(location, "proxied=true") {
-		t.Fatalf("redirect readDeleted Location should include proxied=true, got %q", location)
-	}
-	if strings.Contains(location, "readDeleted=true") {
-		t.Fatalf("redirect readDeleted Location should reflect current query-drop behavior, got %q", location)
-	}
-
-	followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location))
-	_ = framework.ReadAllAndClose(t, followResp)
-	if followResp.StatusCode != http.StatusNotFound {
-		t.Fatalf("redirect-follow without readDeleted query expected 404 for deleted needle, got %d", followResp.StatusCode)
-	}
-}
-
-func TestReadModeRedirectPreservesCollectionQuery(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P1()
-	profile.ReadMode = "redirect"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-
-	const volumeID = uint32(109)
-	const collection = "redirect-collection"
-	framework.AllocateVolume(t, grpc0, volumeID, collection)
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 120006, 0x0102F00D)
-	payload := []byte("redirect-collection-preserve-content")
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	noRedirectClient := &http.Client{
-		Timeout: 10 * time.Second,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-	}
-
-	redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?collection=" + collection
-	var location string
-	if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) {
-		location = resp.Header.Get("Location")
-		_ = framework.ReadAllAndClose(t, resp)
-	}) {
-		t.Fatalf("redirect collection path did not return 301 from non-owning volume server within deadline")
-	}
-	if location == "" {
-		t.Fatalf("redirect collection response missing Location header")
-	}
-	if !strings.Contains(location, "proxied=true") {
-		t.Fatalf("redirect collection Location should include proxied=true, got %q", location)
-	}
-	if !strings.Contains(location, "collection="+collection) {
-		t.Fatalf("redirect collection Location should preserve collection query, got %q", location)
-	}
-
-	followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location))
-	followBody := framework.ReadAllAndClose(t, followResp)
-	if followResp.StatusCode != http.StatusOK {
-		t.Fatalf("redirect-follow expected 200, got %d", followResp.StatusCode)
-	}
-	if string(followBody) != string(payload) {
-		t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload))
-	}
-}
-
-func waitForHTTPStatus(t testing.TB, client *http.Client, url string, expectedStatus int, timeout time.Duration, onMatch func(resp *http.Response)) bool {
-	t.Helper()
-
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url))
-		if resp.StatusCode == expectedStatus {
-			onMatch(resp)
-			return true
-		}
-		_ = framework.ReadAllAndClose(t, resp)
-		time.Sleep(200 * time.Millisecond)
-	}
-
-	return false
-}

test/volume_server/http/read_path_variants_test.go

@@ -1,191 +0,0 @@
-package volume_server_http_test
-
-import (
-	"fmt"
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestReadPathShapesAndIfModifiedSince(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(93)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fullFileID := framework.NewFileID(volumeID, 771234, 0xBEEFCACE)
-	uploadPayload := []byte("read-path-shape-content")
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, uploadPayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	parts := strings.SplitN(fullFileID, ",", 2)
-	if len(parts) != 2 {
-		t.Fatalf("unexpected file id format: %q", fullFileID)
-	}
-	fidOnly := parts[1]
-
-	readByVidFid := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly)))
-	readByVidFidBody := framework.ReadAllAndClose(t, readByVidFid)
-	if readByVidFid.StatusCode != http.StatusOK {
-		t.Fatalf("GET /{vid}/{fid} expected 200, got %d", readByVidFid.StatusCode)
-	}
-	if string(readByVidFidBody) != string(uploadPayload) {
-		t.Fatalf("GET /{vid}/{fid} body mismatch: got %q want %q", string(readByVidFidBody), string(uploadPayload))
-	}
-
-	readWithFilename := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "named.bin")))
-	readWithFilenameBody := framework.ReadAllAndClose(t, readWithFilename)
-	if readWithFilename.StatusCode != http.StatusOK {
-		t.Fatalf("GET /{vid}/{fid}/{filename} expected 200, got %d", readWithFilename.StatusCode)
-	}
-	if string(readWithFilenameBody) != string(uploadPayload) {
-		t.Fatalf("GET /{vid}/{fid}/{filename} body mismatch: got %q want %q", string(readWithFilenameBody), string(uploadPayload))
-	}
-
-	lastModified := readWithFilename.Header.Get("Last-Modified")
-	if lastModified == "" {
-		t.Fatalf("expected Last-Modified header on read response")
-	}
-
-	ifModifiedSinceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fullFileID)
-	ifModifiedSinceReq.Header.Set("If-Modified-Since", lastModified)
-	ifModifiedSinceResp := framework.DoRequest(t, client, ifModifiedSinceReq)
-	_ = framework.ReadAllAndClose(t, ifModifiedSinceResp)
-	if ifModifiedSinceResp.StatusCode != http.StatusNotModified {
-		t.Fatalf("If-Modified-Since expected 304, got %d", ifModifiedSinceResp.StatusCode)
-	}
-
-	headIfModifiedSinceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fullFileID)
-	headIfModifiedSinceReq.Header.Set("If-Modified-Since", lastModified)
-	headIfModifiedSinceResp := framework.DoRequest(t, client, headIfModifiedSinceReq)
-	headIfModifiedSinceBody := framework.ReadAllAndClose(t, headIfModifiedSinceResp)
-	if headIfModifiedSinceResp.StatusCode != http.StatusNotModified {
-		t.Fatalf("HEAD If-Modified-Since expected 304, got %d", headIfModifiedSinceResp.StatusCode)
-	}
-	if len(headIfModifiedSinceBody) != 0 {
-		t.Fatalf("HEAD If-Modified-Since expected empty body, got %d bytes", len(headIfModifiedSinceBody))
-	}
-}
-
-func TestMalformedVidFidPathReturnsBadRequest(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	client := framework.NewHTTPClient()
-
-	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid/not-a-fid"))
-	_ = framework.ReadAllAndClose(t, resp)
-	if resp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("malformed /{vid}/{fid} expected 400, got %d", resp.StatusCode)
-	}
-}
-
-func TestReadWrongCookieReturnsNotFound(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(95)
-	const needleID = uint64(771235)
-	const cookie = uint32(0xBEEFCACF)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, needleID, cookie)
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-cookie-mismatch-content"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1)
-	getResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), wrongCookieFid)
-	_ = framework.ReadAllAndClose(t, getResp)
-	if getResp.StatusCode != http.StatusNotFound {
-		t.Fatalf("GET with wrong cookie expected 404, got %d", getResp.StatusCode)
-	}
-
-	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid))
-	headBody := framework.ReadAllAndClose(t, headResp)
-	if headResp.StatusCode != http.StatusNotFound {
-		t.Fatalf("HEAD with wrong cookie expected 404, got %d", headResp.StatusCode)
-	}
-	if len(headBody) != 0 {
-		t.Fatalf("HEAD wrong-cookie response body should be empty, got %d bytes", len(headBody))
-	}
-}
-
-func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(99)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 772002, 0x2B3C4D5E)
-	payload := []byte("conditional-precedence-content")
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	baselineResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
-	_ = framework.ReadAllAndClose(t, baselineResp)
-	if baselineResp.StatusCode != http.StatusOK {
-		t.Fatalf("baseline read expected 200, got %d", baselineResp.StatusCode)
-	}
-	lastModified := baselineResp.Header.Get("Last-Modified")
-	if lastModified == "" {
-		t.Fatalf("baseline read expected Last-Modified header")
-	}
-
-	precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	precedenceReq.Header.Set("If-Modified-Since", lastModified)
-	precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
-	precedenceResp := framework.DoRequest(t, client, precedenceReq)
-	precedenceBody := framework.ReadAllAndClose(t, precedenceResp)
-	if precedenceResp.StatusCode != http.StatusNotModified {
-		t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode)
-	}
-	if len(precedenceBody) != 0 {
-		t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody))
-	}
-
-	invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
-	invalidIMSReq.Header.Set("If-Modified-Since", "not-a-valid-http-date")
-	invalidIMSReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
-	invalidIMSResp := framework.DoRequest(t, client, invalidIMSReq)
-	invalidIMSBody := framework.ReadAllAndClose(t, invalidIMSResp)
-	if invalidIMSResp.StatusCode != http.StatusOK {
-		t.Fatalf("invalid If-Modified-Since with mismatched etag expected 200, got %d", invalidIMSResp.StatusCode)
-	}
-	if string(invalidIMSBody) != string(payload) {
-		t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload))
-	}
-}

test/volume_server/http/read_write_delete_test.go

@@ -1,123 +0,0 @@
-package volume_server_http_test
-
-import (
-	"net/http"
-	"strconv"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestUploadReadRangeHeadDeleteRoundTrip(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(7)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 123456, 0xA1B2C3D4)
-	data := []byte("hello-volume-server-integration")
-	client := framework.NewHTTPClient()
-
-	uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
-	getBody := framework.ReadAllAndClose(t, getResp)
-	if getResp.StatusCode != http.StatusOK {
-		t.Fatalf("get status: expected 200, got %d", getResp.StatusCode)
-	}
-	if string(getBody) != string(data) {
-		t.Fatalf("get body mismatch: got %q want %q", string(getBody), string(data))
-	}
-	etag := getResp.Header.Get("ETag")
-	if etag == "" {
-		t.Fatalf("expected ETag header from GET response")
-	}
-
-	notModifiedReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid)
-	notModifiedReq.Header.Set("If-None-Match", etag)
-	notModifiedResp := framework.DoRequest(t, client, notModifiedReq)
-	_ = framework.ReadAllAndClose(t, notModifiedResp)
-	if notModifiedResp.StatusCode != http.StatusNotModified {
-		t.Fatalf("if-none-match expected 304, got %d", notModifiedResp.StatusCode)
-	}
-
-	rangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid)
-	rangeReq.Header.Set("Range", "bytes=0-4")
-	rangeResp := framework.DoRequest(t, client, rangeReq)
-	rangeBody := framework.ReadAllAndClose(t, rangeResp)
-	if rangeResp.StatusCode != http.StatusPartialContent {
-		t.Fatalf("range status: expected 206, got %d", rangeResp.StatusCode)
-	}
-	if got, want := string(rangeBody), "hello"; got != want {
-		t.Fatalf("range body mismatch: got %q want %q", got, want)
-	}
-
-	invalidRangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid)
-	invalidRangeReq.Header.Set("Range", "bytes=9999-10000")
-	invalidRangeResp := framework.DoRequest(t, client, invalidRangeReq)
-	_ = framework.ReadAllAndClose(t, invalidRangeResp)
-	if invalidRangeResp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
-		t.Fatalf("invalid range expected 416, got %d", invalidRangeResp.StatusCode)
-	}
-
-	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid))
-	headBody := framework.ReadAllAndClose(t, headResp)
-	if headResp.StatusCode != http.StatusOK {
-		t.Fatalf("head status: expected 200, got %d", headResp.StatusCode)
-	}
-	if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(data)) {
-		t.Fatalf("head content-length mismatch: got %q want %d", got, len(data))
-	}
-	if len(headBody) != 0 {
-		t.Fatalf("head body should be empty, got %d bytes", len(headBody))
-	}
-
-	headNotModifiedReq := mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid)
-	headNotModifiedReq.Header.Set("If-None-Match", etag)
-	headNotModifiedResp := framework.DoRequest(t, client, headNotModifiedReq)
-	headNotModifiedBody := framework.ReadAllAndClose(t, headNotModifiedResp)
-	if headNotModifiedResp.StatusCode != http.StatusNotModified {
-		t.Fatalf("head if-none-match expected 304, got %d", headNotModifiedResp.StatusCode)
-	}
-	if len(headNotModifiedBody) != 0 {
-		t.Fatalf("head if-none-match body should be empty, got %d bytes", len(headNotModifiedBody))
-	}
-
-	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, cluster.VolumeAdminURL()+"/"+fid))
-	_ = framework.ReadAllAndClose(t, deleteResp)
-	if deleteResp.StatusCode != http.StatusAccepted {
-		t.Fatalf("delete status: expected 202, got %d", deleteResp.StatusCode)
-	}
-
-	notFoundResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
-	_ = framework.ReadAllAndClose(t, notFoundResp)
-	if notFoundResp.StatusCode != http.StatusNotFound {
-		t.Fatalf("read after delete: expected 404, got %d", notFoundResp.StatusCode)
-	}
-}
-
-func TestInvalidReadPathReturnsBadRequest(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
-	client := framework.NewHTTPClient()
-
-	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/invalid,needle"))
-	_ = framework.ReadAllAndClose(t, resp)
-	if resp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("invalid read expected 400, got %d", resp.StatusCode)
-	}
-}

test/volume_server/http/throttling_test.go

@@ -1,730 +0,0 @@
-package volume_server_http_test
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
-)
-
-type pausableReader struct {
-	remaining  int64
-	pauseAfter int64
-	paused     bool
-	unblock    <-chan struct{}
-}
-
-func (r *pausableReader) Read(p []byte) (int, error) {
-	if r.remaining <= 0 {
-		return 0, io.EOF
-	}
-	if !r.paused && r.pauseAfter > 0 {
-		n := int64(len(p))
-		if n > r.pauseAfter {
-			n = r.pauseAfter
-		}
-		for i := int64(0); i < n; i++ {
-			p[i] = 'a'
-		}
-		r.remaining -= n
-		r.pauseAfter -= n
-		if r.pauseAfter == 0 {
-			r.paused = true
-		}
-		return int(n), nil
-	}
-	if r.paused {
-		<-r.unblock
-		r.paused = false
-	}
-	n := int64(len(p))
-	if n > r.remaining {
-		n = r.remaining
-	}
-	for i := int64(0); i < n; i++ {
-		p[i] = 'b'
-	}
-	r.remaining -= n
-	return int(n), nil
-}
-
-func TestUploadLimitTimeoutAndReplicateBypass(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(98)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	const blockedUploadSize = 2 * 1024 * 1024 // over 1MB P8 upload limit
-
-	unblockFirstUpload := make(chan struct{})
-	firstUploadDone := make(chan error, 1)
-	firstFID := framework.NewFileID(volumeID, 880001, 0x1A2B3C4D)
-	go func() {
-		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
-			remaining:  blockedUploadSize,
-			pauseAfter: 1,
-			unblock:    unblockFirstUpload,
-		})
-		if err != nil {
-			firstUploadDone <- err
-			return
-		}
-		req.Header.Set("Content-Type", "application/octet-stream")
-		req.ContentLength = blockedUploadSize
-
-		resp, err := (&http.Client{}).Do(req)
-		if resp != nil {
-			_, _ = io.Copy(io.Discard, resp.Body)
-			_ = resp.Body.Close()
-		}
-		firstUploadDone <- err
-	}()
-
-	// Give the first upload time to pass limit checks and block in body processing.
-	time.Sleep(300 * time.Millisecond)
-
-	replicateFID := framework.NewFileID(volumeID, 880002, 0x5E6F7A8B)
-	replicateReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+replicateFID+"?type=replicate", bytes.NewReader([]byte("replicate")))
-	if err != nil {
-		t.Fatalf("create replicate request: %v", err)
-	}
-	replicateReq.Header.Set("Content-Type", "application/octet-stream")
-	replicateReq.ContentLength = int64(len("replicate"))
-	replicateResp, err := framework.NewHTTPClient().Do(replicateReq)
-	if err != nil {
-		t.Fatalf("replicate request failed: %v", err)
-	}
-	_ = framework.ReadAllAndClose(t, replicateResp)
-	if replicateResp.StatusCode != http.StatusCreated {
-		t.Fatalf("replicate request expected 201 bypassing limit, got %d", replicateResp.StatusCode)
-	}
-
-	normalFID := framework.NewFileID(volumeID, 880003, 0x9C0D1E2F)
-	normalReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+normalFID, bytes.NewReader([]byte("normal")))
-	if err != nil {
-		t.Fatalf("create normal request: %v", err)
-	}
-	normalReq.Header.Set("Content-Type", "application/octet-stream")
-	normalReq.ContentLength = int64(len("normal"))
-
-	timeoutClient := &http.Client{Timeout: 10 * time.Second}
-	normalResp, err := timeoutClient.Do(normalReq)
-	if err != nil {
-		t.Fatalf("normal upload request failed: %v", err)
-	}
-	_ = framework.ReadAllAndClose(t, normalResp)
-	if normalResp.StatusCode != http.StatusTooManyRequests {
-		t.Fatalf("normal upload expected 429 while limit blocked, got %d", normalResp.StatusCode)
-	}
-
-	close(unblockFirstUpload)
-	select {
-	case <-firstUploadDone:
-	case <-time.After(5 * time.Second):
-		t.Fatalf("timed out waiting for blocked upload to finish")
-	}
-}
-
-func TestUploadLimitWaitThenProceed(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(111)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	const blockedUploadSize = 2 * 1024 * 1024
-
-	unblockFirstUpload := make(chan struct{})
-	firstUploadDone := make(chan error, 1)
-	firstFID := framework.NewFileID(volumeID, 880601, 0x6A2B3C4D)
-	go func() {
-		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
-			remaining:  blockedUploadSize,
-			pauseAfter: 1,
-			unblock:    unblockFirstUpload,
-		})
-		if err != nil {
-			firstUploadDone <- err
-			return
-		}
-		req.Header.Set("Content-Type", "application/octet-stream")
-		req.ContentLength = blockedUploadSize
-
-		resp, err := (&http.Client{}).Do(req)
-		if resp != nil {
-			_ = framework.ReadAllAndClose(t, resp)
-		}
-		firstUploadDone <- err
-	}()
-
-	time.Sleep(300 * time.Millisecond)
-
-	type uploadResult struct {
-		resp *http.Response
-		err  error
-	}
-	secondUploadDone := make(chan uploadResult, 1)
-	secondFID := framework.NewFileID(volumeID, 880602, 0x6A2B3C4E)
-	go func() {
-		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+secondFID, bytes.NewReader([]byte("wait-then-proceed")))
-		if err != nil {
-			secondUploadDone <- uploadResult{err: err}
-			return
-		}
-		req.Header.Set("Content-Type", "application/octet-stream")
-		req.ContentLength = int64(len("wait-then-proceed"))
-		resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req)
-		secondUploadDone <- uploadResult{resp: resp, err: err}
-	}()
-
-	time.Sleep(500 * time.Millisecond)
-	close(unblockFirstUpload)
-
-	select {
-	case firstErr := <-firstUploadDone:
-		if firstErr != nil {
-			t.Fatalf("first blocked upload failed: %v", firstErr)
-		}
-	case <-time.After(5 * time.Second):
-		t.Fatalf("timed out waiting for first upload completion")
-	}
-
-	select {
-	case result := <-secondUploadDone:
-		if result.err != nil {
-			t.Fatalf("second upload failed: %v", result.err)
-		}
-		_ = framework.ReadAllAndClose(t, result.resp)
-		if result.resp.StatusCode != http.StatusCreated {
-			t.Fatalf("second upload expected 201 after waiting for slot, got %d", result.resp.StatusCode)
-		}
-	case <-time.After(5 * time.Second):
-		t.Fatalf("timed out waiting for second upload completion")
-	}
-}
-
-func TestUploadLimitTimeoutThenRecovery(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(113)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	const blockedUploadSize = 2 * 1024 * 1024
-
-	unblockFirstUpload := make(chan struct{})
-	firstUploadDone := make(chan error, 1)
-	firstFID := framework.NewFileID(volumeID, 880801, 0x7A2B3C4D)
-	go func() {
-		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
-			remaining:  blockedUploadSize,
-			pauseAfter: 1,
-			unblock:    unblockFirstUpload,
-		})
-		if err != nil {
-			firstUploadDone <- err
-			return
-		}
-		req.Header.Set("Content-Type", "application/octet-stream")
-		req.ContentLength = blockedUploadSize
-		resp, err := (&http.Client{}).Do(req)
-		if resp != nil {
-			_ = framework.ReadAllAndClose(t, resp)
-		}
-		firstUploadDone <- err
-	}()
-
-	time.Sleep(300 * time.Millisecond)
-
-	timeoutFID := framework.NewFileID(volumeID, 880802, 0x7A2B3C4E)
-	timeoutResp := framework.UploadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), timeoutFID, []byte("should-timeout"))
-	_ = framework.ReadAllAndClose(t, timeoutResp)
-	if timeoutResp.StatusCode != http.StatusTooManyRequests {
-		t.Fatalf("second upload under blocked pressure expected 429, got %d", timeoutResp.StatusCode)
-	}
-
-	close(unblockFirstUpload)
-	select {
-	case firstErr := <-firstUploadDone:
-		if firstErr != nil {
-			t.Fatalf("first blocked upload failed: %v", firstErr)
-		}
-	case <-time.After(5 * time.Second):
-		t.Fatalf("timed out waiting for first upload completion")
-	}
-
-	recoveryFID := framework.NewFileID(volumeID, 880803, 0x7A2B3C4F)
-	recoveryResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), recoveryFID, []byte("recovered-upload"))
-	_ = framework.ReadAllAndClose(t, recoveryResp)
-	if recoveryResp.StatusCode != http.StatusCreated {
-		t.Fatalf("recovery upload expected 201, got %d", recoveryResp.StatusCode)
-	}
-}
-
-func TestDownloadLimitTimeoutReturnsTooManyRequests(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(99)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	largePayload := make([]byte, 12*1024*1024) // over 1MB P8 download limit
-	for i := range largePayload {
-		largePayload[i] = byte(i % 251)
-	}
-	downloadFID := framework.NewFileID(volumeID, 880101, 0x10203040)
-	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), downloadFID, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID))
-	if err != nil {
-		t.Fatalf("first GET failed: %v", err)
-	}
-	if firstResp.StatusCode != http.StatusOK {
-		_ = framework.ReadAllAndClose(t, firstResp)
-		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
-	}
-	defer firstResp.Body.Close()
-
-	// Keep first response body unread so server write path stays in-flight.
-	time.Sleep(300 * time.Millisecond)
-
-	secondClient := &http.Client{Timeout: 10 * time.Second}
-	secondResp, err := secondClient.Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID))
-	if err != nil {
-		t.Fatalf("second GET failed: %v", err)
-	}
-	_ = framework.ReadAllAndClose(t, secondResp)
-	if secondResp.StatusCode != http.StatusTooManyRequests {
-		t.Fatalf("second GET expected 429 while first download holds limit, got %d", secondResp.StatusCode)
-	}
-}
-
-func TestDownloadLimitWaitThenProceedWithoutReplica(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(112)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	largePayload := make([]byte, 12*1024*1024)
-	for i := range largePayload {
-		largePayload[i] = byte(i % 251)
-	}
-	fid := framework.NewFileID(volumeID, 880701, 0x60708090)
-	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
-	if err != nil {
-		t.Fatalf("first GET failed: %v", err)
-	}
-	if firstResp.StatusCode != http.StatusOK {
-		_ = framework.ReadAllAndClose(t, firstResp)
-		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
-	}
-
-	type readResult struct {
-		resp *http.Response
-		err  error
-	}
-	secondReadDone := make(chan readResult, 1)
-	go func() {
-		resp, readErr := (&http.Client{Timeout: 10 * time.Second}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
-		secondReadDone <- readResult{resp: resp, err: readErr}
-	}()
-
-	time.Sleep(500 * time.Millisecond)
-	_ = firstResp.Body.Close()
-
-	select {
-	case result := <-secondReadDone:
-		if result.err != nil {
-			t.Fatalf("second GET failed: %v", result.err)
-		}
-		secondBody := framework.ReadAllAndClose(t, result.resp)
-		if result.resp.StatusCode != http.StatusOK {
-			t.Fatalf("second GET expected 200 after waiting for slot, got %d", result.resp.StatusCode)
-		}
-		if len(secondBody) != len(largePayload) {
-			t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload))
-		}
-	case <-time.After(5 * time.Second):
-		t.Fatalf("timed out waiting for second GET completion")
-	}
-}
-
-func TestDownloadLimitTimeoutThenRecovery(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(114)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	largePayload := make([]byte, 12*1024*1024)
-	for i := range largePayload {
-		largePayload[i] = byte(i % 251)
-	}
-	fid := framework.NewFileID(volumeID, 880901, 0x708090A0)
-	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
-	if err != nil {
-		t.Fatalf("first GET failed: %v", err)
-	}
-	if firstResp.StatusCode != http.StatusOK {
-		_ = framework.ReadAllAndClose(t, firstResp)
-		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
-	}
-
-	time.Sleep(300 * time.Millisecond)
-
-	timeoutResp := framework.ReadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), fid)
-	_ = framework.ReadAllAndClose(t, timeoutResp)
-	if timeoutResp.StatusCode != http.StatusTooManyRequests {
-		t.Fatalf("second GET under blocked pressure expected 429, got %d", timeoutResp.StatusCode)
-	}
-
-	_ = firstResp.Body.Close()
-
-	recoveryResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid)
-	recoveryBody := framework.ReadAllAndClose(t, recoveryResp)
-	if recoveryResp.StatusCode != http.StatusOK {
-		t.Fatalf("recovery GET expected 200, got %d", recoveryResp.StatusCode)
-	}
-	if len(recoveryBody) != len(largePayload) {
-		t.Fatalf("recovery GET body size mismatch: got %d want %d", len(recoveryBody), len(largePayload))
-	}
-}
-
-func TestDownloadLimitOverageProxiesToReplica(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P8()
-	profile.ReadMode = "proxy"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-	conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
-	defer conn1.Close()
-
-	const volumeID = uint32(100)
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	req := &volume_server_pb.AllocateVolumeRequest{
-		VolumeId:    volumeID,
-		Replication: "001",
-		Version:     uint32(needle.GetCurrentVersion()),
-	}
-	if _, err := grpc0.AllocateVolume(ctx, req); err != nil {
-		t.Fatalf("allocate replicated volume on node0: %v", err)
-	}
-	if _, err := grpc1.AllocateVolume(ctx, req); err != nil {
-		t.Fatalf("allocate replicated volume on node1: %v", err)
-	}
-
-	largePayload := make([]byte, 12*1024*1024)
-	for i := range largePayload {
-		largePayload[i] = byte(i % 251)
-	}
-	fid := framework.NewFileID(volumeID, 880201, 0x0A0B0C0D)
-	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
-	if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
-		_ = framework.ReadAllAndClose(t, resp)
-	}) {
-		t.Fatalf("replica did not become readable within deadline")
-	}
-
-	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid))
-	if err != nil {
-		t.Fatalf("first GET failed: %v", err)
-	}
-	if firstResp.StatusCode != http.StatusOK {
-		_ = framework.ReadAllAndClose(t, firstResp)
-		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
-	}
-	defer firstResp.Body.Close()
-
-	time.Sleep(300 * time.Millisecond)
-
-	secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid))
-	if err != nil {
-		t.Fatalf("second GET failed: %v", err)
-	}
-	secondBody := framework.ReadAllAndClose(t, secondResp)
-	if secondResp.StatusCode != http.StatusOK {
-		t.Fatalf("second GET expected 200 via replica proxy fallback, got %d", secondResp.StatusCode)
-	}
-	if len(secondBody) != len(largePayload) {
-		t.Fatalf("second GET proxied body size mismatch: got %d want %d", len(secondBody), len(largePayload))
-	}
-}
-
-func TestDownloadLimitProxiedRequestSkipsReplicaFallbackAndTimesOut(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	profile := matrix.P8()
-	profile.ReadMode = "proxy"
-	clusterHarness := framework.StartDualVolumeCluster(t, profile)
-
-	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
-	defer conn0.Close()
-	conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
-	defer conn1.Close()
-
-	const volumeID = uint32(106)
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	req := &volume_server_pb.AllocateVolumeRequest{
-		VolumeId:    volumeID,
-		Replication: "001",
-		Version:     uint32(needle.GetCurrentVersion()),
-	}
-	if _, err := grpc0.AllocateVolume(ctx, req); err != nil {
-		t.Fatalf("allocate replicated volume on node0: %v", err)
-	}
-	if _, err := grpc1.AllocateVolume(ctx, req); err != nil {
-		t.Fatalf("allocate replicated volume on node1: %v", err)
-	}
-
-	largePayload := make([]byte, 12*1024*1024)
-	for i := range largePayload {
-		largePayload[i] = byte(i % 251)
-	}
-	fid := framework.NewFileID(volumeID, 880202, 0x0A0B0D0E)
-	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	// Ensure replica path is actually available, so a non-proxied request would proxy.
-	replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
-	if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
-		_ = framework.ReadAllAndClose(t, resp)
-	}) {
-		t.Fatalf("replica did not become readable within deadline")
-	}
-
-	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid))
-	if err != nil {
-		t.Fatalf("first GET failed: %v", err)
-	}
-	if firstResp.StatusCode != http.StatusOK {
-		_ = framework.ReadAllAndClose(t, firstResp)
-		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
-	}
-	defer firstResp.Body.Close()
-
-	time.Sleep(300 * time.Millisecond)
-
-	// proxied=true should bypass replica fallback and hit wait/timeout branch.
-	secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid+"?proxied=true"))
-	if err != nil {
-		t.Fatalf("second GET failed: %v", err)
-	}
-	_ = framework.ReadAllAndClose(t, secondResp)
-	if secondResp.StatusCode != http.StatusTooManyRequests {
-		t.Fatalf("second GET with proxied=true expected 429 timeout path, got %d", secondResp.StatusCode)
-	}
-}
-
-func TestUploadLimitDisabledAllowsConcurrentUploads(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(107)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	const blockedUploadSize = 2 * 1024 * 1024
-	unblockFirstUpload := make(chan struct{})
-	firstUploadDone := make(chan error, 1)
-	firstFID := framework.NewFileID(volumeID, 880301, 0x1A2B3C5D)
-	go func() {
-		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
-			remaining:  blockedUploadSize,
-			pauseAfter: 1,
-			unblock:    unblockFirstUpload,
-		})
-		if err != nil {
-			firstUploadDone <- err
-			return
-		}
-		req.Header.Set("Content-Type", "application/octet-stream")
-		req.ContentLength = blockedUploadSize
-
-		resp, err := (&http.Client{}).Do(req)
-		if resp != nil {
-			_ = framework.ReadAllAndClose(t, resp)
-		}
-		firstUploadDone <- err
-	}()
-
-	time.Sleep(300 * time.Millisecond)
-
-	secondFID := framework.NewFileID(volumeID, 880302, 0x1A2B3C5E)
-	secondResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), secondFID, []byte("no-limit-second-upload"))
-	_ = framework.ReadAllAndClose(t, secondResp)
-	if secondResp.StatusCode != http.StatusCreated {
-		t.Fatalf("second upload with disabled limit expected 201, got %d", secondResp.StatusCode)
-	}
-
-	close(unblockFirstUpload)
-	select {
-	case <-firstUploadDone:
-	case <-time.After(5 * time.Second):
-		t.Fatalf("timed out waiting for first upload completion")
-	}
-}
-
-func TestDownloadLimitDisabledAllowsConcurrentDownloads(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(108)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	largePayload := make([]byte, 12*1024*1024)
-	for i := range largePayload {
-		largePayload[i] = byte(i % 251)
-	}
-	fid := framework.NewFileID(volumeID, 880401, 0x20304050)
-	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
-	if err != nil {
-		t.Fatalf("first GET failed: %v", err)
-	}
-	if firstResp.StatusCode != http.StatusOK {
-		_ = framework.ReadAllAndClose(t, firstResp)
-		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
-	}
-	defer firstResp.Body.Close()
-
-	time.Sleep(300 * time.Millisecond)
-
-	secondResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid)
-	secondBody := framework.ReadAllAndClose(t, secondResp)
-	if secondResp.StatusCode != http.StatusOK {
-		t.Fatalf("second GET with disabled limit expected 200, got %d", secondResp.StatusCode)
-	}
-	if len(secondBody) != len(largePayload) {
-		t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload))
-	}
-}
-
-func TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(110)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	largePayload := make([]byte, 12*1024*1024)
-	for i := range largePayload {
-		largePayload[i] = byte(i % 251)
-	}
-	fid := framework.NewFileID(volumeID, 880501, 0x50607080)
-	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
-	if err != nil {
-		t.Fatalf("first GET failed: %v", err)
-	}
-	if firstResp.StatusCode != http.StatusOK {
-		_ = framework.ReadAllAndClose(t, firstResp)
-		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
-	}
-	defer firstResp.Body.Close()
-
-	time.Sleep(300 * time.Millisecond)
-
-	invalidReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid,1234567890ab")
-	invalidResp := framework.DoRequest(t, framework.NewHTTPClient(), invalidReq)
-	_ = framework.ReadAllAndClose(t, invalidResp)
-	if invalidResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("invalid vid while over limit expected 400, got %d", invalidResp.StatusCode)
-	}
-}

test/volume_server/http/write_delete_variants_test.go

@@ -1,118 +0,0 @@
-package volume_server_http_test
-
-import (
-	"encoding/json"
-	"net/http"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestWriteUnchangedAndDeleteEdgeVariants(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(87)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	const key = uint64(999001)
-	const cookie = uint32(0xDEADBEEF)
-	fid := framework.NewFileID(volumeID, key, cookie)
-	client := framework.NewHTTPClient()
-	payload := []byte("unchanged-write-content")
-
-	firstUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	firstUploadResp := framework.DoRequest(t, client, firstUpload)
-	_ = framework.ReadAllAndClose(t, firstUploadResp)
-	if firstUploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("first upload expected 201, got %d", firstUploadResp.StatusCode)
-	}
-
-	secondUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
-	secondUploadResp := framework.DoRequest(t, client, secondUpload)
-	_ = framework.ReadAllAndClose(t, secondUploadResp)
-	if secondUploadResp.StatusCode != http.StatusNoContent {
-		t.Fatalf("second unchanged upload expected 204, got %d", secondUploadResp.StatusCode)
-	}
-	if secondUploadResp.Header.Get("ETag") == "" {
-		t.Fatalf("second unchanged upload expected ETag header")
-	}
-
-	wrongCookieFid := framework.NewFileID(volumeID, key, cookie+1)
-	wrongCookieDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid))
-	_ = framework.ReadAllAndClose(t, wrongCookieDelete)
-	if wrongCookieDelete.StatusCode != http.StatusBadRequest {
-		t.Fatalf("delete with mismatched cookie expected 400, got %d", wrongCookieDelete.StatusCode)
-	}
-
-	missingDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+framework.NewFileID(volumeID, key+1, cookie)))
-	missingDeleteBody := framework.ReadAllAndClose(t, missingDelete)
-	if missingDelete.StatusCode != http.StatusNotFound {
-		t.Fatalf("delete missing needle expected 404, got %d", missingDelete.StatusCode)
-	}
-
-	var payloadMap map[string]int64
-	if err := json.Unmarshal(missingDeleteBody, &payloadMap); err != nil {
-		t.Fatalf("decode delete missing response: %v", err)
-	}
-	if payloadMap["size"] != 0 {
-		t.Fatalf("delete missing needle expected size=0, got %d", payloadMap["size"])
-	}
-}
-
-func TestDeleteTimestampOverrideKeepsReadDeletedLastModifiedParity(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(88)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 999002, 0xABCD1234)
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("delete-ts-override"))
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	beforeDeleteResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
-	_ = framework.ReadAllAndClose(t, beforeDeleteResp)
-	if beforeDeleteResp.StatusCode != http.StatusOK {
-		t.Fatalf("pre-delete read expected 200, got %d", beforeDeleteResp.StatusCode)
-	}
-	lastModifiedBeforeDelete := beforeDeleteResp.Header.Get("Last-Modified")
-	if lastModifiedBeforeDelete == "" {
-		t.Fatalf("expected Last-Modified before delete")
-	}
-
-	deleteReq := mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid+"?ts=1700000000")
-	deleteResp := framework.DoRequest(t, client, deleteReq)
-	_ = framework.ReadAllAndClose(t, deleteResp)
-	if deleteResp.StatusCode != http.StatusAccepted {
-		t.Fatalf("delete with ts override expected 202, got %d", deleteResp.StatusCode)
-	}
-
-	readDeletedResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true"))
-	_ = framework.ReadAllAndClose(t, readDeletedResp)
-	if readDeletedResp.StatusCode != http.StatusOK {
-		t.Fatalf("readDeleted after ts override expected 200, got %d", readDeletedResp.StatusCode)
-	}
-	lastModified := readDeletedResp.Header.Get("Last-Modified")
-	if lastModified == "" {
-		t.Fatalf("expected Last-Modified header on readDeleted response")
-	}
-	if lastModified != lastModifiedBeforeDelete {
-		t.Fatalf("expected readDeleted Last-Modified parity with pre-delete header, got %q want %q", lastModified, lastModifiedBeforeDelete)
-	}
-}

test/volume_server/http/write_error_variants_test.go

@@ -1,74 +0,0 @@
-package volume_server_http_test
-
-import (
-	"net/http"
-	"strings"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
-	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
-)
-
-func TestWriteInvalidVidAndFidReturnBadRequest(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	client := framework.NewHTTPClient()
-
-	invalidVidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/invalid,12345678", []byte("x"))
-	invalidVidResp := framework.DoRequest(t, client, invalidVidReq)
-	_ = framework.ReadAllAndClose(t, invalidVidResp)
-	if invalidVidResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("write with invalid vid expected 400, got %d", invalidVidResp.StatusCode)
-	}
-
-	invalidFidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/1,bad", []byte("x"))
-	invalidFidResp := framework.DoRequest(t, client, invalidFidReq)
-	_ = framework.ReadAllAndClose(t, invalidFidResp)
-	if invalidFidResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("write with invalid fid expected 400, got %d", invalidFidResp.StatusCode)
-	}
-}
-
-func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(98)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	client := framework.NewHTTPClient()
-	fid := framework.NewFileID(volumeID, 772001, 0x1A2B3C4D)
-
-	malformedMultipartReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, strings.NewReader("not-a-valid-multipart-body"))
-	if err != nil {
-		t.Fatalf("create malformed multipart request: %v", err)
-	}
-	malformedMultipartReq.Header.Set("Content-Type", "multipart/form-data")
-	malformedMultipartResp := framework.DoRequest(t, client, malformedMultipartReq)
-	malformedMultipartBody := framework.ReadAllAndClose(t, malformedMultipartResp)
-	if malformedMultipartResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("malformed multipart write expected 400, got %d", malformedMultipartResp.StatusCode)
-	}
-	if !strings.Contains(strings.ToLower(string(malformedMultipartBody)), "boundary") {
-		t.Fatalf("malformed multipart response should mention boundary parse failure, got %q", string(malformedMultipartBody))
-	}
-
-	md5MismatchReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, []byte("content-md5-mismatch-body"))
-	md5MismatchReq.Header.Set("Content-MD5", "AAAAAAAAAAAAAAAAAAAAAA==")
-	md5MismatchResp := framework.DoRequest(t, client, md5MismatchReq)
-	md5MismatchBody := framework.ReadAllAndClose(t, md5MismatchResp)
-	if md5MismatchResp.StatusCode != http.StatusBadRequest {
-		t.Fatalf("content-md5 mismatch write expected 400, got %d", md5MismatchResp.StatusCode)
-	}
-	if !strings.Contains(string(md5MismatchBody), "Content-MD5") {
-		t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody))
-	}
-}

test/volume_server/matrix/config_profiles.go

@@ -1,63 +0,0 @@
-package matrix
-
-import "time"
-
-// Profile describes one runtime test matrix configuration.
-type Profile struct {
-	Name string
-
-	ReadMode        string
-	SplitPublicPort bool
-
-	EnableJWT      bool
-	JWTSigningKey  string
-	JWTReadKey     string
-	EnableMaintain bool
-
-	ConcurrentUploadLimitMB   int
-	ConcurrentDownloadLimitMB int
-	InflightUploadTimeout     time.Duration
-	InflightDownloadTimeout   time.Duration
-
-	ReplicatedLayout bool
-	HasErasureCoding bool
-	HasRemoteTier    bool
-}
-
-// P1 is the baseline profile: one volume server, no JWT, proxy read mode.
-func P1() Profile {
-	return Profile{
-		Name:            "P1",
-		ReadMode:        "proxy",
-		SplitPublicPort: false,
-	}
-}
-
-// P2 uses split public/admin ports to verify public read-only behavior.
-func P2() Profile {
-	p := P1()
-	p.Name = "P2"
-	p.SplitPublicPort = true
-	return p
-}
-
-// P3 enables JWT verification for read/write flows.
-func P3() Profile {
-	p := P1()
-	p.Name = "P3"
-	p.EnableJWT = true
-	p.JWTSigningKey = "volume-server-write-key"
-	p.JWTReadKey = "volume-server-read-key"
-	return p
-}
-
-// P8 enables upload/download throttling branches.
-func P8() Profile {
-	p := P1()
-	p.Name = "P8"
-	p.ConcurrentUploadLimitMB = 1
-	p.ConcurrentDownloadLimitMB = 1
-	p.InflightUploadTimeout = 2 * time.Second
-	p.InflightDownloadTimeout = 2 * time.Second
-	return p
-}

weed/admin/Makefile

@@ -23,7 +23,7 @@ install-templ:
 .PHONY: generate
 generate: install-templ
 	@echo "Generating templ files..."
-	@cd $(BUILD_DIR) && templ generate
+	@templ generate
 	@echo "Generated: $(TEMPL_GO_FILES)"
 
 # Clean generated files
@@ -37,7 +37,7 @@ clean-templ:
 .PHONY: watch
 watch: install-templ
 	@echo "Watching for templ file changes..."
-	@cd $(BUILD_DIR) && templ generate --watch
+	@templ generate --watch
 
 # Build the main weed binary with admin component
 .PHONY: build
@@ -160,4 +160,4 @@ $(WEED_BINARY): $(TEMPL_GO_FILES) $(GO_FILES)
 # Auto-generate templ files when .templ files change
 %_templ.go: %.templ
 	@echo "Regenerating $@ from $<"
-	@cd $(BUILD_DIR) && templ generate
+	@templ generate