Update cockroachdb-parser to fix 32-bit builds

Updated to seaweedfs/cockroachdb-parser v0.0.0-20251021182748-d0c58c67297e
which includes fixes for building on 32-bit architectures like OpenBSD ARM.

Fixes:
- Integer overflow in tsearch evaluation (int -> int64)
- Flag type overflow in lexbase and tree packages (int -> int64)

.github/workflows/codeql.yml

@@ -22,7 +22,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v4
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: go
@@ -30,7 +30,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below).
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+        uses: github/codeql-action/autobuild@v4
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -44,4 +44,4 @@ jobs:
       #     make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v4

.github/workflows/container_dev.yml

@@ -42,14 +42,14 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_latest.yml

@@ -43,14 +43,14 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_release1.yml

@@ -41,7 +41,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_release2.yml

@@ -42,7 +42,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_release3.yml

@@ -42,7 +42,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -53,6 +53,8 @@ jobs:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.rocksdb_large
+          build-args: |
+            BRANCH=${{ github.sha }}
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release4.yml

@@ -41,7 +41,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_release5.yml

@@ -41,7 +41,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_rocksdb_version.yml

@@ -0,0 +1,110 @@
+name: "docker: build rocksdb image by version"
+
+on:
+  workflow_dispatch:
+    inputs:
+      rocksdb_version:
+        description: 'RocksDB git tag or branch to build (e.g. v10.5.1)'
+        required: true
+        default: 'v10.5.1'
+      seaweedfs_ref:
+        description: 'SeaweedFS git tag, branch, or commit to build'
+        required: true
+        default: 'master'
+      image_tag:
+        description: 'Optional Docker tag suffix (defaults to rocksdb_<rocksdb>_seaweedfs_<ref>)'
+        required: false
+        default: ''
+
+permissions:
+  contents: read
+
+jobs:
+  build-rocksdb-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
+
+      - name: Prepare Docker tag
+        id: tag
+        env:
+          ROCKSDB_VERSION_INPUT: ${{ inputs.rocksdb_version }}
+          SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
+          CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
+        run: |
+          set -euo pipefail
+          sanitize() {
+            local value="$1"
+            value="${value,,}"
+            value="${value// /-}"
+            value="${value//[^a-z0-9_.-]/-}"
+            value="${value#-}"
+            value="${value%-}"
+            printf '%s' "$value"
+          }
+          version="${ROCKSDB_VERSION_INPUT}"
+          seaweed="${SEAWEEDFS_REF_INPUT}"
+          tag="${CUSTOM_TAG_INPUT}"
+          if [ -z "$version" ]; then
+            echo "RocksDB version input is required." >&2
+            exit 1
+          fi
+          if [ -z "$seaweed" ]; then
+            echo "SeaweedFS ref input is required." >&2
+            exit 1
+          fi
+          sanitized_version="$(sanitize "$version")"
+          if [ -z "$sanitized_version" ]; then
+            echo "Unable to sanitize RocksDB version '$version'." >&2
+            exit 1
+          fi
+          sanitized_seaweed="$(sanitize "$seaweed")"
+          if [ -z "$sanitized_seaweed" ]; then
+            echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
+            exit 1
+          fi
+          if [ -z "$tag" ]; then
+            tag="rocksdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
+          fi
+          tag="${tag,,}"
+          tag="${tag// /-}"
+          tag="${tag//[^a-z0-9_.-]/-}"
+          tag="${tag#-}"
+          tag="${tag%-}"
+          if [ -z "$tag" ]; then
+            echo "Resulting Docker tag is empty." >&2
+            exit 1
+          fi
+          echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
+          echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
+          echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
+        with:
+          context: ./docker
+          push: true
+          file: ./docker/Dockerfile.rocksdb_large
+          build-args: |
+            ROCKSDB_VERSION=${{ inputs.rocksdb_version }}
+            BRANCH=${{ inputs.seaweedfs_ref }}
+          platforms: linux/amd64
+          tags: ${{ steps.tag.outputs.full_image }}
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu

.github/workflows/deploy_telemetry.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: '1.24'
 

.github/workflows/depsreview.yml

@@ -11,4 +11,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@bc41886e18ea39df68b1b1245f4184881938e050
+        uses: actions/dependency-review-action@40c09b7dc99638e5ddb0bfd91c1673effc064d8a

.github/workflows/e2e.yml

@@ -24,7 +24,7 @@ jobs:
     timeout-minutes: 30
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@8e57b58e57be52ac95949151e2777ffda8501267 # v2
+      uses: actions/setup-go@c0137caad775660c0844396c52da96e560aba63d # v2
       with:
         go-version: ^1.13
       id: go
@@ -32,14 +32,54 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
 
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      
+    - name: Cache Docker layers
+      uses: actions/cache@v4
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-e2e-
+
     - name: Install dependencies
       run: |
-        sudo apt-get update
-        sudo apt-get install -y fuse
+        # Use faster mirrors and install with timeout
+        echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee /etc/apt/sources.list
+        echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
+        
+        sudo apt-get update --fix-missing
+        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends fuse
+        
+        # Verify FUSE installation
+        echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')"
+        echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
 
     - name: Start SeaweedFS
-      timeout-minutes: 5
-      run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
+      timeout-minutes: 10
+      run: |
+        # Enable Docker buildkit for better caching
+        export DOCKER_BUILDKIT=1
+        export COMPOSE_DOCKER_CLI_BUILD=1
+        
+        # Build with retry logic
+        for i in {1..3}; do
+          echo "Build attempt $i/3"
+          if make build_e2e; then
+            echo "Build successful on attempt $i"
+            break
+          elif [ $i -eq 3 ]; then
+            echo "Build failed after 3 attempts"
+            exit 1
+          else
+            echo "Build attempt $i failed, retrying in 30 seconds..."
+            sleep 30
+          fi
+        done
+        
+        # Start services with wait
+        docker compose -f ./compose/e2e-mount.yml up --wait
 
     - name: Run FIO 4k
       timeout-minutes: 15

.github/workflows/fuse-integration.yml

@@ -22,7 +22,7 @@ permissions:
   contents: read
 
 env:
-  GO_VERSION: '1.21'
+  GO_VERSION: '1.24'
   TEST_TIMEOUT: '45m'
 
 jobs:
@@ -36,7 +36,7 @@ jobs:
       uses: actions/checkout@v5
       
     - name: Set up Go ${{ env.GO_VERSION }}
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v6
       with:
         go-version: ${{ env.GO_VERSION }}
         

.github/workflows/go.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@8e57b58e57be52ac95949151e2777ffda8501267 # v2
+      uses: actions/setup-go@c0137caad775660c0844396c52da96e560aba63d # v2
       with:
         go-version: ^1.13
       id: go

.github/workflows/helm_chart_release.yml

@@ -20,4 +20,4 @@ jobs:
           charts_dir: k8s/charts
           target_dir: helm
           branch: gh-pages
-          helm_version: v3.18.4
+          helm_version: "3.18.4"

.github/workflows/helm_ci.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           version: v3.18.4
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: '3.9'
           check-latest: true

.github/workflows/kafka-quicktest.yml

@@ -0,0 +1,124 @@
+name: "Kafka Quick Test (Load Test with Schema Registry)"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:  # Allow manual trigger
+
+concurrency:
+  group: ${{ github.head_ref }}/kafka-quicktest
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  kafka-client-quicktest:
+    name: Kafka Client Load Test (Quick)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+        cache: true
+        cache-dependency-path: |
+          **/go.sum
+      id: go
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Install dependencies
+      run: |
+        # Ensure make is available
+        sudo apt-get update -qq
+        sudo apt-get install -y make
+
+    - name: Validate test setup
+      working-directory: test/kafka/kafka-client-loadtest
+      run: |
+        make validate-setup
+
+    - name: Run quick-test
+      working-directory: test/kafka/kafka-client-loadtest
+      run: |
+        # Run the quick-test target which includes:
+        # 1. Building the gateway
+        # 2. Starting all services (SeaweedFS, MQ broker, Schema Registry)
+        # 3. Registering Avro schemas
+        # 4. Running a 1-minute load test with Avro messages
+        # Override GOARCH to build for AMD64 (GitHub Actions runners are x86_64)
+        GOARCH=amd64 make quick-test
+      env:
+        # Docker Compose settings
+        COMPOSE_HTTP_TIMEOUT: 300
+        DOCKER_CLIENT_TIMEOUT: 300
+        # Test parameters (set by quick-test, but can override)
+        TEST_DURATION: 60s
+        PRODUCER_COUNT: 1
+        CONSUMER_COUNT: 1
+        MESSAGE_RATE: 10
+        VALUE_TYPE: avro
+
+    - name: Show test results
+      if: always()
+      working-directory: test/kafka/kafka-client-loadtest
+      run: |
+        echo "========================================="
+        echo "Test Results"
+        echo "========================================="
+        make show-results || echo "Could not retrieve results"
+
+    - name: Show service logs on failure
+      if: failure()
+      working-directory: test/kafka/kafka-client-loadtest
+      run: |
+        echo "========================================="
+        echo "Service Logs"
+        echo "========================================="
+        
+        echo "Checking running containers..."
+        docker compose ps || true
+        
+        echo "========================================="
+        echo "Master Logs"
+        echo "========================================="
+        docker compose logs --tail=100 seaweedfs-master 2>&1 || echo "No master logs available"
+        
+        echo "========================================="
+        echo "MQ Broker Logs (Last 100 lines)"
+        echo "========================================="
+        docker compose logs --tail=100 seaweedfs-mq-broker 2>&1 || echo "No broker logs available"
+        
+        echo "========================================="
+        echo "Kafka Gateway Logs (FULL - Critical for debugging)"
+        echo "========================================="
+        docker compose logs kafka-gateway 2>&1 || echo "ERROR: Could not retrieve kafka-gateway logs"
+        
+        echo "========================================="
+        echo "Schema Registry Logs (FULL)"
+        echo "========================================="
+        docker compose logs schema-registry 2>&1 || echo "ERROR: Could not retrieve schema-registry logs"
+        
+        echo "========================================="
+        echo "Load Test Logs"
+        echo "========================================="
+        docker compose logs --tail=100 kafka-client-loadtest 2>&1 || echo "No loadtest logs available"
+
+    - name: Cleanup
+      if: always()
+      working-directory: test/kafka/kafka-client-loadtest
+      run: |
+        # Stop containers first
+        docker compose --profile loadtest --profile monitoring down -v --remove-orphans || true
+        # Clean up data with sudo to handle Docker root-owned files
+        sudo rm -rf data/* || true
+        # Clean up binary
+        rm -f weed-linux-* || true

.github/workflows/kafka-tests.yml

@@ -0,0 +1,814 @@
+name: "Kafka Gateway Tests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+concurrency:
+  group: ${{ github.head_ref }}/kafka-tests
+  cancel-in-progress: true
+
+# Force different runners for better isolation
+env:
+  FORCE_RUNNER_SEPARATION: true
+
+permissions:
+  contents: read
+
+jobs:
+  kafka-unit-tests:
+    name: Kafka Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        container-id: [unit-tests-1]
+    container:
+      image: golang:1.24-alpine
+      options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }}
+    env:
+      GOMAXPROCS: 1
+      CGO_ENABLED: 0
+      CONTAINER_ID: ${{ matrix.container-id }}
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+      id: go
+
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Setup Container Environment
+      run: |
+        apk add --no-cache git
+        ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
+        
+    - name: Get dependencies
+      run: |
+        cd test/kafka
+        go mod download
+
+    - name: Run Kafka Gateway Unit Tests
+      run: |
+        cd test/kafka
+        # Set process limits for container isolation
+        ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
+        ulimit -u 100 || echo "Warning: Could not set process limit"
+        go test -v -timeout 10s ./unit/...
+
+  kafka-integration-tests:
+    name: Kafka Integration Tests (Critical)
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        container-id: [integration-1]
+    container:
+      image: golang:1.24-alpine
+      options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }}
+    env:
+      GOMAXPROCS: 2
+      CGO_ENABLED: 0
+      KAFKA_TEST_ISOLATION: "true"
+      CONTAINER_ID: ${{ matrix.container-id }}
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+      id: go
+
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Setup Integration Container Environment
+      run: |
+        apk add --no-cache git procps
+        ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
+        
+    - name: Get dependencies
+      run: |
+        cd test/kafka
+        go mod download
+
+    - name: Run Integration Tests
+      run: |
+        cd test/kafka
+        # Higher limits for integration tests
+        ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
+        ulimit -u 200 || echo "Warning: Could not set process limit"
+        go test -v -timeout 90s ./integration/...
+      env:
+        GOMAXPROCS: 2
+
+  kafka-e2e-tests:
+    name: Kafka End-to-End Tests (with SMQ)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        container-id: [e2e-1]
+    container:
+      image: golang:1.24-alpine
+      options: --cpus 2.0 --memory 2g --hostname kafka-e2e-${{ matrix.container-id }}
+    env:
+      GOMAXPROCS: 2
+      CGO_ENABLED: 0
+      KAFKA_E2E_ISOLATION: "true"
+      CONTAINER_ID: ${{ matrix.container-id }}
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+        cache: true
+        cache-dependency-path: |
+          **/go.sum
+      id: go
+
+    - name: Setup E2E Container Environment
+      run: |
+        apk add --no-cache git procps curl netcat-openbsd
+        ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
+        
+    - name: Warm Go module cache
+      run: |
+        # Warm cache for root module
+        go mod download || true
+        # Warm cache for kafka test module
+        cd test/kafka
+        go mod download || true
+
+    - name: Get dependencies
+      run: |
+        cd test/kafka
+        # Use go mod download with timeout to prevent hanging
+        timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
+
+    - name: Build and start SeaweedFS MQ
+      run: |
+        set -e
+        cd $GITHUB_WORKSPACE
+        # Build weed binary
+        go build -o /usr/local/bin/weed ./weed
+        # Start SeaweedFS components with MQ brokers
+        export WEED_DATA_DIR=/tmp/seaweedfs-e2e-$RANDOM
+        mkdir -p "$WEED_DATA_DIR"
+        
+        # Start SeaweedFS server (master, volume, filer) with consistent IP advertising
+        nohup weed -v 1 server \
+          -ip="127.0.0.1" \
+          -ip.bind="0.0.0.0" \
+          -dir="$WEED_DATA_DIR" \
+          -master.raftHashicorp \
+          -master.port=9333 \
+          -volume.port=8081 \
+          -filer.port=8888 \
+          -filer=true \
+          -metricsPort=9325 \
+          > /tmp/weed-server.log 2>&1 &
+          
+        # Wait for master to be ready
+        for i in $(seq 1 30); do
+          if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
+            echo "SeaweedFS master HTTP is up"; break
+          fi
+          echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
+        done
+        
+        # Wait for master gRPC to be ready (this is what broker discovery uses)
+        echo "Waiting for master gRPC port..."
+        for i in $(seq 1 30); do
+          if nc -z 127.0.0.1 19333; then
+            echo "✓ SeaweedFS master gRPC is up (port 19333)"
+            break
+          fi
+          echo "  Waiting for master gRPC... ($i/30)"; sleep 1
+        done
+        
+        # Give server time to initialize all components including gRPC services
+        echo "Waiting for SeaweedFS components to initialize..."
+        sleep 15
+        
+        # Additional wait specifically for gRPC services to be ready for streaming
+        echo "Allowing extra time for master gRPC streaming services to initialize..."
+        sleep 10
+        
+        # Start MQ broker with maximum verbosity for debugging
+        echo "Starting MQ broker..."
+        nohup weed -v 3 mq.broker \
+          -master="127.0.0.1:9333" \
+          -ip="127.0.0.1" \
+          -port=17777 \
+          -logFlushInterval=0 \
+          > /tmp/weed-mq-broker.log 2>&1 &
+        
+        # Wait for broker to be ready with better error reporting  
+        sleep 15
+        broker_ready=false
+        for i in $(seq 1 20); do
+          if nc -z 127.0.0.1 17777; then
+            echo "SeaweedFS MQ broker is up"
+            broker_ready=true
+            break
+          fi
+          echo "Waiting for MQ broker... ($i/20)"; sleep 1
+        done
+        
+        # Give broker additional time to register with master
+        if [ "$broker_ready" = true ]; then
+          echo "Allowing broker to register with master..."
+          sleep 30
+          
+          # Check if broker is properly registered by querying cluster nodes
+          echo "Cluster status after broker registration:"
+          curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
+          
+          echo "Checking cluster topology (includes registered components):"
+          curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
+          
+          echo "Verifying broker discovery via master client debug:"
+          echo "If broker registration is successful, it should appear in dir status"
+          
+          echo "Testing gRPC connectivity with weed binary:"
+          echo "This simulates what the gateway does during broker discovery..."
+          timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
+          echo "Shell test results:"
+          cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
+        fi
+        
+        # Check if broker failed to start and show logs
+        if [ "$broker_ready" = false ]; then
+          echo "ERROR: MQ broker failed to start. Broker logs:"
+          cat /tmp/weed-mq-broker.log || echo "No broker logs found"
+          echo "Server logs:"  
+          tail -20 /tmp/weed-server.log || echo "No server logs found"
+          exit 1
+        fi
+
+    - name: Run End-to-End Tests
+      run: |
+        cd test/kafka
+        # Higher limits for E2E tests
+        ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
+        ulimit -u 200 || echo "Warning: Could not set process limit"
+        
+        # Allow additional time for all background processes to settle
+        echo "Allowing additional settlement time for SeaweedFS ecosystem..."
+        sleep 15
+        
+        # Run tests and capture result
+        if ! go test -v -timeout 180s ./e2e/...; then
+          echo "========================================="
+          echo "Tests failed! Showing debug information:"
+          echo "========================================="
+          echo "Server logs (last 50 lines):"
+          tail -50 /tmp/weed-server.log || echo "No server logs"
+          echo "========================================="
+          echo "Broker logs (last 50 lines):"
+          tail -50 /tmp/weed-mq-broker.log || echo "No broker logs"
+          echo "========================================="
+          exit 1
+        fi
+      env:
+        GOMAXPROCS: 2
+        SEAWEEDFS_MASTERS: 127.0.0.1:9333
+
+  kafka-consumer-group-tests:
+    name: Kafka Consumer Group Tests (Highly Isolated)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        container-id: [consumer-group-1]
+    container:
+      image: golang:1.24-alpine
+      options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }}
+    env:
+      GOMAXPROCS: 1
+      CGO_ENABLED: 0
+      KAFKA_CONSUMER_ISOLATION: "true"
+      CONTAINER_ID: ${{ matrix.container-id }}
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+        cache: true
+        cache-dependency-path: |
+          **/go.sum
+      id: go
+
+    - name: Setup Consumer Group Container Environment
+      run: |
+        apk add --no-cache git procps curl netcat-openbsd
+        ulimit -n 256 || echo "Warning: Could not set file descriptor limit"
+        
+    - name: Warm Go module cache
+      run: |
+        # Warm cache for root module
+        go mod download || true
+        # Warm cache for kafka test module
+        cd test/kafka
+        go mod download || true
+
+    - name: Get dependencies
+      run: |
+        cd test/kafka
+        # Use go mod download with timeout to prevent hanging
+        timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
+
+    - name: Build and start SeaweedFS MQ
+      run: |
+        set -e
+        cd $GITHUB_WORKSPACE
+        # Build weed binary
+        go build -o /usr/local/bin/weed ./weed
+        # Start SeaweedFS components with MQ brokers
+        export WEED_DATA_DIR=/tmp/seaweedfs-mq-$RANDOM
+        mkdir -p "$WEED_DATA_DIR"
+        
+        # Start SeaweedFS server (master, volume, filer) with consistent IP advertising
+        nohup weed -v 1 server \
+          -ip="127.0.0.1" \
+          -ip.bind="0.0.0.0" \
+          -dir="$WEED_DATA_DIR" \
+          -master.raftHashicorp \
+          -master.port=9333 \
+          -volume.port=8081 \
+          -filer.port=8888 \
+          -filer=true \
+          -metricsPort=9325 \
+          > /tmp/weed-server.log 2>&1 &
+          
+        # Wait for master to be ready
+        for i in $(seq 1 30); do
+          if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
+            echo "SeaweedFS master HTTP is up"; break
+          fi
+          echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
+        done
+        
+        # Wait for master gRPC to be ready (this is what broker discovery uses)
+        echo "Waiting for master gRPC port..."
+        for i in $(seq 1 30); do
+          if nc -z 127.0.0.1 19333; then
+            echo "✓ SeaweedFS master gRPC is up (port 19333)"
+            break
+          fi
+          echo "  Waiting for master gRPC... ($i/30)"; sleep 1
+        done
+        
+        # Give server time to initialize all components including gRPC services
+        echo "Waiting for SeaweedFS components to initialize..."
+        sleep 15
+        
+        # Additional wait specifically for gRPC services to be ready for streaming
+        echo "Allowing extra time for master gRPC streaming services to initialize..."
+        sleep 10
+        
+        # Start MQ broker with maximum verbosity for debugging
+        echo "Starting MQ broker..."
+        nohup weed -v 3 mq.broker \
+          -master="127.0.0.1:9333" \
+          -ip="127.0.0.1" \
+          -port=17777 \
+          -logFlushInterval=0 \
+          > /tmp/weed-mq-broker.log 2>&1 &
+        
+        # Wait for broker to be ready with better error reporting  
+        sleep 15
+        broker_ready=false
+        for i in $(seq 1 20); do
+          if nc -z 127.0.0.1 17777; then
+            echo "SeaweedFS MQ broker is up"
+            broker_ready=true
+            break
+          fi
+          echo "Waiting for MQ broker... ($i/20)"; sleep 1
+        done
+        
+        # Give broker additional time to register with master
+        if [ "$broker_ready" = true ]; then
+          echo "Allowing broker to register with master..."
+          sleep 30
+          
+          # Check if broker is properly registered by querying cluster nodes
+          echo "Cluster status after broker registration:"
+          curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
+          
+          echo "Checking cluster topology (includes registered components):"
+          curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
+          
+          echo "Verifying broker discovery via master client debug:"
+          echo "If broker registration is successful, it should appear in dir status"
+          
+          echo "Testing gRPC connectivity with weed binary:"
+          echo "This simulates what the gateway does during broker discovery..."
+          timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
+          echo "Shell test results:"
+          cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
+        fi
+        
+        # Check if broker failed to start and show logs
+        if [ "$broker_ready" = false ]; then
+          echo "ERROR: MQ broker failed to start. Broker logs:"
+          cat /tmp/weed-mq-broker.log || echo "No broker logs found"
+          echo "Server logs:"  
+          tail -20 /tmp/weed-server.log || echo "No server logs found"
+          exit 1
+        fi
+
+    - name: Run Consumer Group Tests
+      run: |
+        cd test/kafka
+        # Test consumer group functionality with explicit timeout
+        ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
+        ulimit -u 100 || echo "Warning: Could not set process limit"
+        timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
+      env:
+        GOMAXPROCS: 1
+        SEAWEEDFS_MASTERS: 127.0.0.1:9333
+
+  kafka-client-compatibility:
+    name: Kafka Client Compatibility (with SMQ)
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        container-id: [client-compat-1]
+    container:
+      image: golang:1.24-alpine
+      options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }}
+    env:
+      GOMAXPROCS: 1
+      CGO_ENABLED: 0
+      KAFKA_CLIENT_ISOLATION: "true"
+      CONTAINER_ID: ${{ matrix.container-id }}
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+        cache: true
+        cache-dependency-path: |
+          **/go.sum
+      id: go
+
+    - name: Setup Client Container Environment
+      run: |
+        apk add --no-cache git procps curl netcat-openbsd
+        ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
+
+    - name: Warm Go module cache
+      run: |
+        # Warm cache for root module
+        go mod download || true
+        # Warm cache for kafka test module
+        cd test/kafka
+        go mod download || true
+
+    - name: Get dependencies
+      run: |
+        cd test/kafka
+        timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
+
+    - name: Build and start SeaweedFS MQ
+      run: |
+        set -e
+        cd $GITHUB_WORKSPACE
+        # Build weed binary
+        go build -o /usr/local/bin/weed ./weed
+        # Start SeaweedFS components with MQ brokers
+        export WEED_DATA_DIR=/tmp/seaweedfs-client-$RANDOM
+        mkdir -p "$WEED_DATA_DIR"
+        
+        # Start SeaweedFS server (master, volume, filer) with consistent IP advertising
+        nohup weed -v 1 server \
+          -ip="127.0.0.1" \
+          -ip.bind="0.0.0.0" \
+          -dir="$WEED_DATA_DIR" \
+          -master.raftHashicorp \
+          -master.port=9333 \
+          -volume.port=8081 \
+          -filer.port=8888 \
+          -filer=true \
+          -metricsPort=9325 \
+          > /tmp/weed-server.log 2>&1 &
+          
+        # Wait for master to be ready
+        for i in $(seq 1 30); do
+          if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
+            echo "SeaweedFS master HTTP is up"; break
+          fi
+          echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
+        done
+        
+        # Wait for master gRPC to be ready (this is what broker discovery uses)
+        echo "Waiting for master gRPC port..."
+        for i in $(seq 1 30); do
+          if nc -z 127.0.0.1 19333; then
+            echo "✓ SeaweedFS master gRPC is up (port 19333)"
+            break
+          fi
+          echo "  Waiting for master gRPC... ($i/30)"; sleep 1
+        done
+        
+        # Give server time to initialize all components including gRPC services
+        echo "Waiting for SeaweedFS components to initialize..."
+        sleep 15
+        
+        # Additional wait specifically for gRPC services to be ready for streaming
+        echo "Allowing extra time for master gRPC streaming services to initialize..."
+        sleep 10
+        
+        # Start MQ broker with maximum verbosity for debugging
+        echo "Starting MQ broker..."
+        nohup weed -v 3 mq.broker \
+          -master="127.0.0.1:9333" \
+          -ip="127.0.0.1" \
+          -port=17777 \
+          -logFlushInterval=0 \
+          > /tmp/weed-mq-broker.log 2>&1 &
+        
+        # Wait for broker to be ready with better error reporting  
+        sleep 15
+        broker_ready=false
+        for i in $(seq 1 20); do
+          if nc -z 127.0.0.1 17777; then
+            echo "SeaweedFS MQ broker is up"
+            broker_ready=true
+            break
+          fi
+          echo "Waiting for MQ broker... ($i/20)"; sleep 1
+        done
+        
+        # Give broker additional time to register with master
+        if [ "$broker_ready" = true ]; then
+          echo "Allowing broker to register with master..."
+          sleep 30
+          
+          # Check if broker is properly registered by querying cluster nodes
+          echo "Cluster status after broker registration:"
+          curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
+          
+          echo "Checking cluster topology (includes registered components):"
+          curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
+          
+          echo "Verifying broker discovery via master client debug:"
+          echo "If broker registration is successful, it should appear in dir status"
+          
+          echo "Testing gRPC connectivity with weed binary:"
+          echo "This simulates what the gateway does during broker discovery..."
+          timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
+          echo "Shell test results:"
+          cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
+        fi
+        
+        # Check if broker failed to start and show logs
+        if [ "$broker_ready" = false ]; then
+          echo "ERROR: MQ broker failed to start. Broker logs:"
+          cat /tmp/weed-mq-broker.log || echo "No broker logs found"
+          echo "Server logs:"  
+          tail -20 /tmp/weed-server.log || echo "No server logs found"
+          exit 1
+        fi
+
+    - name: Run Client Compatibility Tests
+      run: |
+        cd test/kafka
+        go test -v -run "^TestClientCompatibility" -timeout 180s ./integration/...
+      env:
+        GOMAXPROCS: 1
+        SEAWEEDFS_MASTERS: 127.0.0.1:9333
+
+  kafka-smq-integration-tests:
+    name: Kafka SMQ Integration Tests (Full Stack)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        container-id: [smq-integration-1]
+    container:
+      image: golang:1.24-alpine
+      options: --cpus 1.0 --memory 2g --hostname kafka-smq-${{ matrix.container-id }}
+    env:
+      GOMAXPROCS: 1
+      CGO_ENABLED: 0
+      KAFKA_SMQ_INTEGRATION: "true"
+      CONTAINER_ID: ${{ matrix.container-id }}
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+        cache: true
+        cache-dependency-path: |
+          **/go.sum
+      id: go
+
+    - name: Setup SMQ Integration Container Environment
+      run: |
+        apk add --no-cache git procps curl netcat-openbsd
+        ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
+        
+    - name: Warm Go module cache
+      run: |
+        # Warm cache for root module
+        go mod download || true
+        # Warm cache for kafka test module
+        cd test/kafka
+        go mod download || true
+
+    - name: Get dependencies
+      run: |
+        cd test/kafka
+        timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
+
+    - name: Build and start SeaweedFS MQ
+      run: |
+        set -e
+        cd $GITHUB_WORKSPACE
+        # Build weed binary
+        go build -o /usr/local/bin/weed ./weed
+        # Start SeaweedFS components with MQ brokers
+        export WEED_DATA_DIR=/tmp/seaweedfs-smq-$RANDOM
+        mkdir -p "$WEED_DATA_DIR"
+        
+        # Start SeaweedFS server (master, volume, filer) with consistent IP advertising
+        nohup weed -v 1 server \
+          -ip="127.0.0.1" \
+          -ip.bind="0.0.0.0" \
+          -dir="$WEED_DATA_DIR" \
+          -master.raftHashicorp \
+          -master.port=9333 \
+          -volume.port=8081 \
+          -filer.port=8888 \
+          -filer=true \
+          -metricsPort=9325 \
+          > /tmp/weed-server.log 2>&1 &
+          
+        # Wait for master to be ready
+        for i in $(seq 1 30); do
+          if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
+            echo "SeaweedFS master HTTP is up"; break
+          fi
+          echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
+        done
+        
+        # Wait for master gRPC to be ready (this is what broker discovery uses)
+        echo "Waiting for master gRPC port..."
+        for i in $(seq 1 30); do
+          if nc -z 127.0.0.1 19333; then
+            echo "✓ SeaweedFS master gRPC is up (port 19333)"
+            break
+          fi
+          echo "  Waiting for master gRPC... ($i/30)"; sleep 1
+        done
+        
+        # Give server time to initialize all components including gRPC services
+        echo "Waiting for SeaweedFS components to initialize..."
+        sleep 15
+        
+        # Additional wait specifically for gRPC services to be ready for streaming
+        echo "Allowing extra time for master gRPC streaming services to initialize..."
+        sleep 10
+        
+        # Start MQ broker with maximum verbosity for debugging
+        echo "Starting MQ broker..."
+        nohup weed -v 3 mq.broker \
+          -master="127.0.0.1:9333" \
+          -ip="127.0.0.1" \
+          -port=17777 \
+          -logFlushInterval=0 \
+          > /tmp/weed-mq-broker.log 2>&1 &
+        
+        # Wait for broker to be ready with better error reporting  
+        sleep 15
+        broker_ready=false
+        for i in $(seq 1 20); do
+          if nc -z 127.0.0.1 17777; then
+            echo "SeaweedFS MQ broker is up"
+            broker_ready=true
+            break
+          fi
+          echo "Waiting for MQ broker... ($i/20)"; sleep 1
+        done
+        
+        # Give broker additional time to register with master
+        if [ "$broker_ready" = true ]; then
+          echo "Allowing broker to register with master..."
+          sleep 30
+          
+          # Check if broker is properly registered by querying cluster nodes
+          echo "Cluster status after broker registration:"
+          curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
+          
+          echo "Checking cluster topology (includes registered components):"
+          curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
+          
+          echo "Verifying broker discovery via master client debug:"
+          echo "If broker registration is successful, it should appear in dir status"
+          
+          echo "Testing gRPC connectivity with weed binary:"
+          echo "This simulates what the gateway does during broker discovery..."
+          timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
+          echo "Shell test results:"
+          cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
+        fi
+        
+        # Check if broker failed to start and show logs
+        if [ "$broker_ready" = false ]; then
+          echo "ERROR: MQ broker failed to start. Broker logs:"
+          cat /tmp/weed-mq-broker.log || echo "No broker logs found"
+          echo "Server logs:"  
+          tail -20 /tmp/weed-server.log || echo "No server logs found"
+          exit 1
+        fi
+
+    - name: Run SMQ Integration Tests
+      run: |
+        cd test/kafka
+        ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
+        ulimit -u 100 || echo "Warning: Could not set process limit"
+        # Run the dedicated SMQ integration tests
+        go test -v -run "^TestSMQIntegration" -timeout 180s ./integration/...
+      env:
+        GOMAXPROCS: 1
+        SEAWEEDFS_MASTERS: 127.0.0.1:9333
+
+  kafka-protocol-tests:
+    name: Kafka Protocol Tests (Isolated)
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        container-id: [protocol-1]
+    container:
+      image: golang:1.24-alpine
+      options: --cpus 1.0 --memory 1g --tmpfs /tmp:exec --hostname kafka-protocol-${{ matrix.container-id }}
+    env:
+      GOMAXPROCS: 1
+      CGO_ENABLED: 0
+      KAFKA_PROTOCOL_ISOLATION: "true"
+      CONTAINER_ID: ${{ matrix.container-id }}
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+      id: go
+
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Setup Protocol Container Environment
+      run: |
+        apk add --no-cache git procps
+        # Ensure proper permissions for test execution
+        chmod -R 755 /tmp || true
+        export TMPDIR=/tmp
+        export GOCACHE=/tmp/go-cache
+        mkdir -p $GOCACHE
+        chmod 755 $GOCACHE
+
+    - name: Get dependencies
+      run: |
+        cd test/kafka
+        go mod download
+
+    - name: Run Protocol Tests
+      run: |
+        cd test/kafka
+        export TMPDIR=/tmp
+        export GOCACHE=/tmp/go-cache
+        # Run protocol tests from the weed/mq/kafka directory since they test the protocol implementation
+        cd ../../weed/mq/kafka
+        go test -v -run "^Test.*" -timeout 10s ./...
+      env:
+        GOMAXPROCS: 1
+        TMPDIR: /tmp
+        GOCACHE: /tmp/go-cache

.github/workflows/postgres-tests.yml

@@ -0,0 +1,73 @@
+name: "PostgreSQL Gateway Tests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+concurrency:
+  group: ${{ github.head_ref }}/postgres-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  postgres-basic-tests:
+    name: PostgreSQL Basic Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: test/postgres
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.24
+      id: go
+
+    - name: Check out code
+      uses: actions/checkout@v5
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Cache Docker layers
+      uses: actions/cache@v4
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-postgres-
+
+    - name: Start PostgreSQL Gateway Services
+      run: |
+        make dev-start
+        sleep 10
+
+    - name: Run Basic Connectivity Test
+      run: |
+        make test-basic
+
+    - name: Run PostgreSQL Client Tests
+      run: |
+        make test-client
+
+    - name: Save logs
+      if: always()
+      run: |
+        docker compose logs > postgres-output.log || true
+
+    - name: Archive logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: postgres-logs
+        path: test/postgres/postgres-output.log
+
+    - name: Cleanup
+      if: always()
+      run: |
+        make clean || true

.github/workflows/s3-go-tests.yml

@@ -28,7 +28,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -92,7 +92,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -140,7 +140,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -191,7 +191,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -258,7 +258,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -322,7 +322,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -373,7 +373,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go

.github/workflows/s3-iam-tests.yml

@@ -38,7 +38,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -87,7 +87,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -179,7 +179,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -239,7 +239,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go

.github/workflows/s3-keycloak-tests.yml

@@ -38,7 +38,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go

.github/workflows/s3-sse-tests.yml

@@ -45,7 +45,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -109,7 +109,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -157,7 +157,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -206,7 +206,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -255,7 +255,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -306,7 +306,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go

.github/workflows/s3tests.yml

@@ -23,13 +23,13 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go 1.x
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.9'
 
@@ -41,6 +41,12 @@ jobs:
           pip install tox
           pip install -e .
 
+      - name: Fix S3 tests bucket creation conflicts
+        run: |
+          python3 test/s3/fix_s3_tests_bucket_conflicts.py
+        env:
+          S3_TESTS_PATH: s3-tests
+
       - name: Run Basic S3 tests
         timeout-minutes: 15
         env:
@@ -58,7 +64,7 @@ jobs:
             -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
             -volume.max=100 -volume.preStopSeconds=1 \
             -master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \
-            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
           pid=$!
           
           # Wait for all SeaweedFS components to be ready
@@ -101,7 +107,7 @@ jobs:
           
           echo "All SeaweedFS components are ready!"
           cd ../s3-tests
-          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
+          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
           
           # Debug: Show the config file contents
           echo "=== S3 Config File Contents ==="
@@ -126,183 +132,183 @@ jobs:
           echo "✅ S3 server is responding, starting tests..."
           
           tox -- \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_many \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_encoding_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_prefix \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_percentage \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_whitespace \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_dot \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_unordered \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_unordered \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_invalid \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_return_data \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
-          s3tests_boto3/functional/test_s3.py::test_bucket_delete_notexist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_create_delete \
-          s3tests_boto3/functional/test_s3.py::test_object_read_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_multi_object_delete \
-          s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete \
-          s3tests_boto3/functional/test_s3.py::test_object_head_zero_bytes \
-          s3tests_boto3/functional/test_s3.py::test_object_write_check_etag \
-          s3tests_boto3/functional/test_s3.py::test_object_write_cache_control \
-          s3tests_boto3/functional/test_s3.py::test_object_write_expires \
-          s3tests_boto3/functional/test_s3.py::test_object_write_read_update_read_delete \
-          s3tests_boto3/functional/test_s3.py::test_object_metadata_replaced_on_put \
-          s3tests_boto3/functional/test_s3.py::test_object_write_file \
-          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format \
-          s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature \
-          s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
-          s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list \
-          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument \
-          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument \
-          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum \
-          s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_good \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_failed \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_failed \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
-          s3tests_boto3/functional/test_s3.py::test_bucket_head \
-          s3tests_boto3/functional/test_s3.py::test_bucket_head_notexist \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_acl \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_gone \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
-          s3tests_boto3/functional/test_s3.py::test_object_anon_put \
-          s3tests_boto3/functional/test_s3.py::test_object_put_authenticated \
-          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
-          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl \
-          s3tests_boto3/functional/test_s3.py::test_buckets_create_then_list \
-          s3tests_boto3/functional/test_s3.py::test_buckets_list_ctime \
-          s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth \
-          s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth \
-          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
-          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_zero_size \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_same_bucket \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_diff_bucket \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_canned_acl \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_bucket_not_found \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_key_not_found \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_small \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_special_names \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \
-          s3tests_boto3/functional/test_s3.py::test_multipart_get_part \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_empty \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \
-          s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_resend_part \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_missing_part \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_incorrect_etag \
-          s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \
-          s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \
-          s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_read_4mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_read_8mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_write_1mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_write_4mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_write_8mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_1mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \
-          s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \
-          s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \
-          s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
-          s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_failed \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_good \
-          s3tests_boto3/functional/test_s3.py::test_lifecycle_set \
-          s3tests_boto3/functional/test_s3.py::test_lifecycle_get \
-          s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter
+          s3tests/functional/test_s3.py::test_bucket_list_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_distinct \
+          s3tests/functional/test_s3.py::test_bucket_list_many \
+          s3tests/functional/test_s3.py::test_bucket_listv2_many \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_encoding_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_encoding_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_percentage \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_whitespace \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_dot \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_none \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_none \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_alt \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_alt \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_none \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_none \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_one \
+          s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_zero \
+          s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_none \
+          s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_none \
+          s3tests/functional/test_s3.py::test_bucket_list_unordered \
+          s3tests/functional/test_s3.py::test_bucket_listv2_unordered \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_invalid \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_none \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken \
+          s3tests/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_not_in_list \
+          s3tests/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_after_list \
+          s3tests/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
+          s3tests/functional/test_s3.py::test_bucket_list_return_data \
+          s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous \
+          s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous \
+          s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
+          s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
+          s3tests/functional/test_s3.py::test_bucket_list_long_name \
+          s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests/functional/test_s3.py::test_bucket_delete_notexist \
+          s3tests/functional/test_s3.py::test_bucket_create_delete \
+          s3tests/functional/test_s3.py::test_object_read_not_exist \
+          s3tests/functional/test_s3.py::test_multi_object_delete \
+          s3tests/functional/test_s3.py::test_multi_objectv2_delete \
+          s3tests/functional/test_s3.py::test_object_head_zero_bytes \
+          s3tests/functional/test_s3.py::test_object_write_check_etag \
+          s3tests/functional/test_s3.py::test_object_write_cache_control \
+          s3tests/functional/test_s3.py::test_object_write_expires \
+          s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
+          s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
+          s3tests/functional/test_s3.py::test_object_write_file \
+          s3tests/functional/test_s3.py::test_post_object_invalid_date_format \
+          s3tests/functional/test_s3.py::test_post_object_no_key_specified \
+          s3tests/functional/test_s3.py::test_post_object_missing_signature \
+          s3tests/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
+          s3tests/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
+          s3tests/functional/test_s3.py::test_post_object_missing_expires_condition \
+          s3tests/functional/test_s3.py::test_post_object_missing_conditions_list \
+          s3tests/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
+          s3tests/functional/test_s3.py::test_post_object_missing_content_length_argument \
+          s3tests/functional/test_s3.py::test_post_object_invalid_content_length_argument \
+          s3tests/functional/test_s3.py::test_post_object_upload_size_below_minimum \
+          s3tests/functional/test_s3.py::test_post_object_empty_conditions \
+          s3tests/functional/test_s3.py::test_get_object_ifmatch_good \
+          s3tests/functional/test_s3.py::test_get_object_ifnonematch_good \
+          s3tests/functional/test_s3.py::test_get_object_ifmatch_failed \
+          s3tests/functional/test_s3.py::test_get_object_ifnonematch_failed \
+          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
+          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_bucket_head \
+          s3tests/functional/test_s3.py::test_bucket_head_notexist \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated_object_acl \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated_object_gone \
+          s3tests/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
+          s3tests/functional/test_s3.py::test_object_anon_put \
+          s3tests/functional/test_s3.py::test_object_put_authenticated \
+          s3tests/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
+          s3tests/functional/test_s3.py::test_bucket_recreate_new_acl \
+          s3tests/functional/test_s3.py::test_buckets_create_then_list \
+          s3tests/functional/test_s3.py::test_buckets_list_ctime \
+          s3tests/functional/test_s3.py::test_list_buckets_invalid_auth \
+          s3tests/functional/test_s3.py::test_list_buckets_bad_auth \
+          s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
+          s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
+          s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests/functional/test_s3.py::test_object_copy_zero_size \
+          s3tests/functional/test_s3.py::test_object_copy_same_bucket \
+          s3tests/functional/test_s3.py::test_object_copy_to_itself \
+          s3tests/functional/test_s3.py::test_object_copy_diff_bucket \
+          s3tests/functional/test_s3.py::test_object_copy_canned_acl \
+          s3tests/functional/test_s3.py::test_object_copy_bucket_not_found \
+          s3tests/functional/test_s3.py::test_object_copy_key_not_found \
+          s3tests/functional/test_s3.py::test_multipart_copy_small \
+          s3tests/functional/test_s3.py::test_multipart_copy_without_range \
+          s3tests/functional/test_s3.py::test_multipart_copy_special_names \
+          s3tests/functional/test_s3.py::test_multipart_copy_multiple_sizes \
+          s3tests/functional/test_s3.py::test_multipart_get_part \
+          s3tests/functional/test_s3.py::test_multipart_upload \
+          s3tests/functional/test_s3.py::test_multipart_upload_empty \
+          s3tests/functional/test_s3.py::test_multipart_upload_multiple_sizes \
+          s3tests/functional/test_s3.py::test_multipart_upload_contents \
+          s3tests/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
+          s3tests/functional/test_s3.py::test_multipart_upload_size_too_small \
+          s3tests/functional/test_s3.py::test_multipart_resend_first_finishes_last \
+          s3tests/functional/test_s3.py::test_multipart_upload_resend_part \
+          s3tests/functional/test_s3.py::test_multipart_upload_missing_part \
+          s3tests/functional/test_s3.py::test_multipart_upload_incorrect_etag \
+          s3tests/functional/test_s3.py::test_abort_multipart_upload \
+          s3tests/functional/test_s3.py::test_list_multipart_upload \
+          s3tests/functional/test_s3.py::test_atomic_read_1mb \
+          s3tests/functional/test_s3.py::test_atomic_read_4mb \
+          s3tests/functional/test_s3.py::test_atomic_read_8mb \
+          s3tests/functional/test_s3.py::test_atomic_write_1mb \
+          s3tests/functional/test_s3.py::test_atomic_write_4mb \
+          s3tests/functional/test_s3.py::test_atomic_write_8mb \
+          s3tests/functional/test_s3.py::test_atomic_dual_write_1mb \
+          s3tests/functional/test_s3.py::test_atomic_dual_write_4mb \
+          s3tests/functional/test_s3.py::test_atomic_dual_write_8mb \
+          s3tests/functional/test_s3.py::test_atomic_multipart_upload_write \
+          s3tests/functional/test_s3.py::test_ranged_request_response_code \
+          s3tests/functional/test_s3.py::test_ranged_big_request_response_code \
+          s3tests/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
+          s3tests/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
+          s3tests/functional/test_s3.py::test_copy_object_ifmatch_good \
+          s3tests/functional/test_s3.py::test_copy_object_ifnonematch_failed \
+          s3tests/functional/test_s3.py::test_copy_object_ifmatch_failed \
+          s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \
+          s3tests/functional/test_s3.py::test_lifecycle_set \
+          s3tests/functional/test_s3.py::test_lifecycle_get \
+          s3tests/functional/test_s3.py::test_lifecycle_set_filter
           kill -9 $pid || true
           # Clean up data directory
           rm -rf "$WEED_DATA_DIR" || true
@@ -316,13 +322,13 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go 1.x
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.9'
 
@@ -334,6 +340,12 @@ jobs:
           pip install tox
           pip install -e .
 
+      - name: Fix S3 tests bucket creation conflicts
+        run: |
+          python3 test/s3/fix_s3_tests_bucket_conflicts.py
+        env:
+          S3_TESTS_PATH: s3-tests
+
       - name: Run S3 Object Lock, Retention, and Versioning tests
         timeout-minutes: 15
         shell: bash
@@ -344,12 +356,16 @@ jobs:
           # Create clean data directory for this test run
           export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-versioning-$(date +%s)"
           mkdir -p "$WEED_DATA_DIR"
+          
+          # Verify S3 config file exists
+          echo "Checking S3 config file: $GITHUB_WORKSPACE/docker/compose/s3.json"
+          ls -la "$GITHUB_WORKSPACE/docker/compose/s3.json"
           weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
             -dir="$WEED_DATA_DIR" \
             -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
             -volume.max=100 -volume.preStopSeconds=1 \
             -master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \
-            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
           pid=$!
           
           # Wait for all SeaweedFS components to be ready
@@ -392,16 +408,15 @@ jobs:
           
           echo "All SeaweedFS components are ready!"
           cd ../s3-tests
-          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
-          # Fix bucket creation conflicts in versioning tests by replacing _create_objects calls
-          sed -i 's/bucket_name = _create_objects(bucket_name=bucket_name,keys=key_names)/# Use the existing bucket for object creation\n    client = get_client()\n    for key in key_names:\n        client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
-          sed -i 's/bucket = _create_objects(bucket_name=bucket_name, keys=key_names)/# Use the existing bucket for object creation\n    client = get_client()\n    for key in key_names:\n        client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
+          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
           # Create and update s3tests.conf to use port 8001
           cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-versioning.conf
           sed -i 's/port = 8000/port = 8001/g' ../docker/compose/s3tests-versioning.conf
           sed -i 's/:8000/:8001/g' ../docker/compose/s3tests-versioning.conf
           sed -i 's/localhost:8000/localhost:8001/g' ../docker/compose/s3tests-versioning.conf
           sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8001/g' ../docker/compose/s3tests-versioning.conf
+          # Use the configured bucket prefix from config and do not override with unique prefixes
+          # This avoids mismatch in tests that rely on a fixed provided name
           export S3TEST_CONF=../docker/compose/s3tests-versioning.conf
           
           # Debug: Show the config file contents
@@ -423,12 +438,45 @@ jobs:
             echo "S3 connection test failed, retrying... ($i/10)"
             sleep 2
           done
-          # tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or (versioning and not test_versioning_obj_suspend_versions and not test_bucket_list_return_data_versioning and not test_versioning_concurrent_multi_object_delete)" --tb=short
-          # Run all versioning and object lock tests including specific list object versions tests
-          tox -- \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_return_data_versioning \
-          s3tests_boto3/functional/test_s3.py::test_versioning_obj_list_marker \
-          s3tests_boto3/functional/test_s3.py -k "object_lock or versioning" --tb=short
+          
+          # Force cleanup any existing buckets to avoid conflicts
+          echo "Cleaning up any existing buckets..."
+          python3 -c "
+          import boto3
+          from botocore.exceptions import ClientError
+          try:
+              s3 = boto3.client('s3', 
+                  endpoint_url='http://localhost:8001',
+                  aws_access_key_id='0555b35654ad1656d804',
+                  aws_secret_access_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==')
+              buckets = s3.list_buckets()['Buckets']
+              for bucket in buckets:
+                  bucket_name = bucket['Name']
+                  print(f'Deleting bucket: {bucket_name}')
+                  try:
+                      # Delete all objects first
+                      objects = s3.list_objects_v2(Bucket=bucket_name)
+                      if 'Contents' in objects:
+                          for obj in objects['Contents']:
+                              s3.delete_object(Bucket=bucket_name, Key=obj['Key'])
+                      # Delete all versions if versioning enabled
+                      versions = s3.list_object_versions(Bucket=bucket_name)
+                      if 'Versions' in versions:
+                          for version in versions['Versions']:
+                              s3.delete_object(Bucket=bucket_name, Key=version['Key'], VersionId=version['VersionId'])
+                      if 'DeleteMarkers' in versions:
+                          for marker in versions['DeleteMarkers']:
+                              s3.delete_object(Bucket=bucket_name, Key=marker['Key'], VersionId=marker['VersionId'])
+                      # Delete bucket
+                      s3.delete_bucket(Bucket=bucket_name)
+                  except ClientError as e:
+                      print(f'Error deleting bucket {bucket_name}: {e}')
+          except Exception as e:
+              print(f'Cleanup failed: {e}')
+          " || echo "Cleanup completed with some errors (expected)"
+          
+          # Run versioning and object lock tests once (avoid duplicates)
+          tox -- s3tests/functional/test_s3.py -k "object_lock or versioning" --tb=short
           kill -9 $pid || true
           # Clean up data directory
           rm -rf "$WEED_DATA_DIR" || true
@@ -442,13 +490,13 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go 1.x
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.9'
 
@@ -475,7 +523,7 @@ jobs:
             -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
             -volume.max=100 -volume.preStopSeconds=1 \
             -master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \
-            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
           pid=$!
           
           # Wait for all SeaweedFS components to be ready
@@ -518,7 +566,7 @@ jobs:
           
           echo "All SeaweedFS components are ready!"
           cd ../s3-tests
-          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
+          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
           # Create and update s3tests.conf to use port 8002
           cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-cors.conf
           sed -i 's/port = 8000/port = 8002/g' ../docker/compose/s3tests-cors.conf
@@ -547,11 +595,11 @@ jobs:
             sleep 2
           done
           # Run CORS-specific tests from s3-tests suite
-          tox -- s3tests_boto3/functional/test_s3.py -k "cors" --tb=short || echo "No CORS tests found in s3-tests suite"
+          tox -- s3tests/functional/test_s3.py -k "cors" --tb=short || echo "No CORS tests found in s3-tests suite"
           # If no specific CORS tests exist, run bucket configuration tests that include CORS
-          tox -- s3tests_boto3/functional/test_s3.py::test_put_bucket_cors || echo "No put_bucket_cors test found"
-          tox -- s3tests_boto3/functional/test_s3.py::test_get_bucket_cors || echo "No get_bucket_cors test found"
-          tox -- s3tests_boto3/functional/test_s3.py::test_delete_bucket_cors || echo "No delete_bucket_cors test found"
+          tox -- s3tests/functional/test_s3.py::test_put_bucket_cors || echo "No put_bucket_cors test found"
+          tox -- s3tests/functional/test_s3.py::test_get_bucket_cors || echo "No get_bucket_cors test found"
+          tox -- s3tests/functional/test_s3.py::test_delete_bucket_cors || echo "No delete_bucket_cors test found"
           kill -9 $pid || true
           # Clean up data directory
           rm -rf "$WEED_DATA_DIR" || true
@@ -565,7 +613,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go 1.x
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
@@ -585,7 +633,7 @@ jobs:
             -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
             -volume.max=100 -volume.preStopSeconds=1 \
             -master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
-            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
           pid=$!
           
           # Wait for all SeaweedFS components to be ready
@@ -665,13 +713,13 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Set up Go 1.x
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
         id: go
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.9'
 
@@ -766,7 +814,7 @@ jobs:
             -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
             -volume.max=100 -volume.preStopSeconds=1 \
             -master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \
-            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json \
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
             > /tmp/seaweedfs-sql-server.log 2>&1 &
           pid=$!
           
@@ -848,7 +896,7 @@ jobs:
           
           echo "All SeaweedFS components are ready!"
           cd ../s3-tests
-          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
+          sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
           # Create and update s3tests.conf to use port 8004
           cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-sql.conf
           sed -i 's/port = 8000/port = 8004/g' ../docker/compose/s3tests-sql.conf
@@ -899,183 +947,183 @@ jobs:
             sleep 2
           done
           tox -- \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_many \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_encoding_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_prefix \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_percentage \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_whitespace \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_dot \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_unordered \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_unordered \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_invalid \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_return_data \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
-          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
-          s3tests_boto3/functional/test_s3.py::test_bucket_delete_notexist \
-          s3tests_boto3/functional/test_s3.py::test_bucket_create_delete \
-          s3tests_boto3/functional/test_s3.py::test_object_read_not_exist \
-          s3tests_boto3/functional/test_s3.py::test_multi_object_delete \
-          s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete \
-          s3tests_boto3/functional/test_s3.py::test_object_head_zero_bytes \
-          s3tests_boto3/functional/test_s3.py::test_object_write_check_etag \
-          s3tests_boto3/functional/test_s3.py::test_object_write_cache_control \
-          s3tests_boto3/functional/test_s3.py::test_object_write_expires \
-          s3tests_boto3/functional/test_s3.py::test_object_write_read_update_read_delete \
-          s3tests_boto3/functional/test_s3.py::test_object_metadata_replaced_on_put \
-          s3tests_boto3/functional/test_s3.py::test_object_write_file \
-          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format \
-          s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature \
-          s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
-          s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list \
-          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
-          s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument \
-          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument \
-          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum \
-          s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_good \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_failed \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_failed \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
-          s3tests_boto3/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
-          s3tests_boto3/functional/test_s3.py::test_bucket_head \
-          s3tests_boto3/functional/test_s3.py::test_bucket_head_notexist \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_acl \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_gone \
-          s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
-          s3tests_boto3/functional/test_s3.py::test_object_anon_put \
-          s3tests_boto3/functional/test_s3.py::test_object_put_authenticated \
-          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
-          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl \
-          s3tests_boto3/functional/test_s3.py::test_buckets_create_then_list \
-          s3tests_boto3/functional/test_s3.py::test_buckets_list_ctime \
-          s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth \
-          s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth \
-          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
-          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
-          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_zero_size \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_same_bucket \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_diff_bucket \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_canned_acl \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_bucket_not_found \
-          s3tests_boto3/functional/test_s3.py::test_object_copy_key_not_found \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_small \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_special_names \
-          s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \
-          s3tests_boto3/functional/test_s3.py::test_multipart_get_part \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_empty \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \
-          s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_resend_part \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_missing_part \
-          s3tests_boto3/functional/test_s3.py::test_multipart_upload_incorrect_etag \
-          s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \
-          s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \
-          s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_read_4mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_read_8mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_write_1mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_write_4mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_write_8mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_1mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \
-          s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \
-          s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \
-          s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \
-          s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
-          s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_failed \
-          s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_good \
-          s3tests_boto3/functional/test_s3.py::test_lifecycle_set \
-          s3tests_boto3/functional/test_s3.py::test_lifecycle_get \
-          s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter
+          s3tests/functional/test_s3.py::test_bucket_list_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_distinct \
+          s3tests/functional/test_s3.py::test_bucket_list_many \
+          s3tests/functional/test_s3.py::test_bucket_listv2_many \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_encoding_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_encoding_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_percentage \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_whitespace \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_dot \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_none \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_none \
+          s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_basic \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_basic \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_alt \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_alt \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_empty \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_none \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_none \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
+          s3tests/functional/test_s3.py::test_bucket_list_prefix_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_one \
+          s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_zero \
+          s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_none \
+          s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_none \
+          s3tests/functional/test_s3.py::test_bucket_list_unordered \
+          s3tests/functional/test_s3.py::test_bucket_listv2_unordered \
+          s3tests/functional/test_s3.py::test_bucket_list_maxkeys_invalid \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_none \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
+          s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken \
+          s3tests/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_not_in_list \
+          s3tests/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
+          s3tests/functional/test_s3.py::test_bucket_list_marker_after_list \
+          s3tests/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
+          s3tests/functional/test_s3.py::test_bucket_list_return_data \
+          s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous \
+          s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous \
+          s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
+          s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
+          s3tests/functional/test_s3.py::test_bucket_list_long_name \
+          s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests/functional/test_s3.py::test_bucket_delete_notexist \
+          s3tests/functional/test_s3.py::test_bucket_create_delete \
+          s3tests/functional/test_s3.py::test_object_read_not_exist \
+          s3tests/functional/test_s3.py::test_multi_object_delete \
+          s3tests/functional/test_s3.py::test_multi_objectv2_delete \
+          s3tests/functional/test_s3.py::test_object_head_zero_bytes \
+          s3tests/functional/test_s3.py::test_object_write_check_etag \
+          s3tests/functional/test_s3.py::test_object_write_cache_control \
+          s3tests/functional/test_s3.py::test_object_write_expires \
+          s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
+          s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
+          s3tests/functional/test_s3.py::test_object_write_file \
+          s3tests/functional/test_s3.py::test_post_object_invalid_date_format \
+          s3tests/functional/test_s3.py::test_post_object_no_key_specified \
+          s3tests/functional/test_s3.py::test_post_object_missing_signature \
+          s3tests/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
+          s3tests/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
+          s3tests/functional/test_s3.py::test_post_object_missing_expires_condition \
+          s3tests/functional/test_s3.py::test_post_object_missing_conditions_list \
+          s3tests/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
+          s3tests/functional/test_s3.py::test_post_object_missing_content_length_argument \
+          s3tests/functional/test_s3.py::test_post_object_invalid_content_length_argument \
+          s3tests/functional/test_s3.py::test_post_object_upload_size_below_minimum \
+          s3tests/functional/test_s3.py::test_post_object_empty_conditions \
+          s3tests/functional/test_s3.py::test_get_object_ifmatch_good \
+          s3tests/functional/test_s3.py::test_get_object_ifnonematch_good \
+          s3tests/functional/test_s3.py::test_get_object_ifmatch_failed \
+          s3tests/functional/test_s3.py::test_get_object_ifnonematch_failed \
+          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
+          s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests/functional/test_s3.py::test_bucket_head \
+          s3tests/functional/test_s3.py::test_bucket_head_notexist \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated_object_acl \
+          s3tests/functional/test_s3.py::test_object_raw_authenticated_object_gone \
+          s3tests/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
+          s3tests/functional/test_s3.py::test_object_anon_put \
+          s3tests/functional/test_s3.py::test_object_put_authenticated \
+          s3tests/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
+          s3tests/functional/test_s3.py::test_bucket_recreate_new_acl \
+          s3tests/functional/test_s3.py::test_buckets_create_then_list \
+          s3tests/functional/test_s3.py::test_buckets_list_ctime \
+          s3tests/functional/test_s3.py::test_list_buckets_invalid_auth \
+          s3tests/functional/test_s3.py::test_list_buckets_bad_auth \
+          s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
+          s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
+          s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests/functional/test_s3.py::test_object_copy_zero_size \
+          s3tests/functional/test_s3.py::test_object_copy_same_bucket \
+          s3tests/functional/test_s3.py::test_object_copy_to_itself \
+          s3tests/functional/test_s3.py::test_object_copy_diff_bucket \
+          s3tests/functional/test_s3.py::test_object_copy_canned_acl \
+          s3tests/functional/test_s3.py::test_object_copy_bucket_not_found \
+          s3tests/functional/test_s3.py::test_object_copy_key_not_found \
+          s3tests/functional/test_s3.py::test_multipart_copy_small \
+          s3tests/functional/test_s3.py::test_multipart_copy_without_range \
+          s3tests/functional/test_s3.py::test_multipart_copy_special_names \
+          s3tests/functional/test_s3.py::test_multipart_copy_multiple_sizes \
+          s3tests/functional/test_s3.py::test_multipart_get_part \
+          s3tests/functional/test_s3.py::test_multipart_upload \
+          s3tests/functional/test_s3.py::test_multipart_upload_empty \
+          s3tests/functional/test_s3.py::test_multipart_upload_multiple_sizes \
+          s3tests/functional/test_s3.py::test_multipart_upload_contents \
+          s3tests/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
+          s3tests/functional/test_s3.py::test_multipart_upload_size_too_small \
+          s3tests/functional/test_s3.py::test_multipart_resend_first_finishes_last \
+          s3tests/functional/test_s3.py::test_multipart_upload_resend_part \
+          s3tests/functional/test_s3.py::test_multipart_upload_missing_part \
+          s3tests/functional/test_s3.py::test_multipart_upload_incorrect_etag \
+          s3tests/functional/test_s3.py::test_abort_multipart_upload \
+          s3tests/functional/test_s3.py::test_list_multipart_upload \
+          s3tests/functional/test_s3.py::test_atomic_read_1mb \
+          s3tests/functional/test_s3.py::test_atomic_read_4mb \
+          s3tests/functional/test_s3.py::test_atomic_read_8mb \
+          s3tests/functional/test_s3.py::test_atomic_write_1mb \
+          s3tests/functional/test_s3.py::test_atomic_write_4mb \
+          s3tests/functional/test_s3.py::test_atomic_write_8mb \
+          s3tests/functional/test_s3.py::test_atomic_dual_write_1mb \
+          s3tests/functional/test_s3.py::test_atomic_dual_write_4mb \
+          s3tests/functional/test_s3.py::test_atomic_dual_write_8mb \
+          s3tests/functional/test_s3.py::test_atomic_multipart_upload_write \
+          s3tests/functional/test_s3.py::test_ranged_request_response_code \
+          s3tests/functional/test_s3.py::test_ranged_big_request_response_code \
+          s3tests/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
+          s3tests/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
+          s3tests/functional/test_s3.py::test_copy_object_ifmatch_good \
+          s3tests/functional/test_s3.py::test_copy_object_ifnonematch_failed \
+          s3tests/functional/test_s3.py::test_copy_object_ifmatch_failed \
+          s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \
+          s3tests/functional/test_s3.py::test_lifecycle_set \
+          s3tests/functional/test_s3.py::test_lifecycle_get \
+          s3tests/functional/test_s3.py::test_lifecycle_set_filter
           kill -9 $pid || true
           # Clean up data directory
           rm -rf "$WEED_DATA_DIR" || true

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
           go-version: ^1.24
 

.gitignore

@@ -123,3 +123,4 @@ ADVANCED_IAM_DEVELOPMENT_PLAN.md
 /test/s3/iam/test-volume-data
 *.log
 weed-iam
+test/kafka/kafka-client-loadtest/weed-linux-arm64

SQL_FEATURE_PLAN.md

@@ -0,0 +1,145 @@
+# SQL Query Engine Feature, Dev, and Test Plan
+
+This document outlines the plan for adding SQL querying support to SeaweedFS, focusing on reading and analyzing data from Message Queue (MQ) topics.
+
+## Feature Plan
+
+**1. Goal**
+
+To provide a SQL querying interface for SeaweedFS, enabling analytics on existing MQ topics. This enables:
+- Basic querying with SELECT, WHERE, aggregations on MQ topics
+- Schema discovery and metadata operations (SHOW DATABASES, SHOW TABLES, DESCRIBE)
+- In-place analytics on Parquet-stored messages without data movement
+
+**2. Key Features**
+
+*   **Schema Discovery and Metadata:**
+    *   `SHOW DATABASES` - List all MQ namespaces
+    *   `SHOW TABLES` - List all topics in a namespace  
+    *   `DESCRIBE table_name` - Show topic schema details
+    *   Automatic schema detection from existing Parquet data
+*   **Basic Query Engine:**
+    *   `SELECT` support with `WHERE`, `LIMIT`, `OFFSET`
+    *   Aggregation functions: `COUNT()`, `SUM()`, `AVG()`, `MIN()`, `MAX()`
+    *   Temporal queries with timestamp-based filtering
+*   **User Interfaces:**
+    *   New CLI command `weed sql` with interactive shell mode
+    *   Optional: Web UI for query execution and result visualization
+*   **Output Formats:**
+    *   JSON (default), CSV, Parquet for result sets
+    *   Streaming results for large queries
+    *   Pagination support for result navigation
+
+## Development Plan
+
+
+
+**3. Data Source Integration**
+
+*   **MQ Topic Connector (Primary):**
+    *   Build on existing `weed/mq/logstore/read_parquet_to_log.go`
+    *   Implement efficient Parquet scanning with predicate pushdown
+    *   Support schema evolution and backward compatibility
+    *   Handle partition-based parallelism for scalable queries
+*   **Schema Registry Integration:**
+    *   Extend `weed/mq/schema/schema.go` for SQL metadata operations
+    *   Read existing topic schemas for query planning
+    *   Handle schema evolution during query execution
+
+**4. API & CLI Integration**
+
+*   **CLI Command:**
+    *   New `weed sql` command with interactive shell mode (similar to `weed shell`)
+    *   Support for script execution and result formatting
+    *   Connection management for remote SeaweedFS clusters
+*   **gRPC API:**
+    *   Add SQL service to existing MQ broker gRPC interface
+    *   Enable efficient query execution with streaming results
+
+## Example Usage Scenarios
+
+**Scenario 1: Schema Discovery and Metadata**
+```sql
+-- List all namespaces (databases)
+SHOW DATABASES;
+
+-- List topics in a namespace
+USE my_namespace;
+SHOW TABLES;
+
+-- View topic structure and discovered schema
+DESCRIBE user_events;
+```
+
+**Scenario 2: Data Querying**
+```sql
+-- Basic filtering and projection
+SELECT user_id, event_type, timestamp 
+FROM user_events 
+WHERE timestamp > 1640995200000 
+LIMIT 100;
+
+-- Aggregation queries  
+SELECT COUNT(*) as event_count
+FROM user_events 
+WHERE timestamp >= 1640995200000;
+
+-- More aggregation examples
+SELECT MAX(timestamp), MIN(timestamp) 
+FROM user_events;
+```
+
+**Scenario 3: Analytics & Monitoring**
+```sql
+-- Basic analytics
+SELECT COUNT(*) as total_events
+FROM user_events 
+WHERE timestamp >= 1640995200000;
+
+-- Simple monitoring
+SELECT AVG(response_time) as avg_response
+FROM api_logs
+WHERE timestamp >= 1640995200000;
+
+## Architecture Overview
+
+```
+SQL Query Flow:
+                                  1. Parse SQL        2. Plan & Optimize      3. Execute Query
+┌─────────────┐    ┌──────────────┐    ┌─────────────────┐    ┌──────────────┐
+│   Client    │    │  SQL Parser  │    │  Query Planner  │    │   Execution  │
+│    (CLI)    │──→ │ PostgreSQL   │──→ │   & Optimizer   │──→ │    Engine    │
+│             │    │ (Custom)     │    │                 │    │              │
+└─────────────┘    └──────────────┘    └─────────────────┘    └──────────────┘
+                                               │                       │
+                                               │ Schema Lookup         │ Data Access
+                                               ▼                       ▼
+                    ┌─────────────────────────────────────────────────────────────┐
+                    │                    Schema Catalog                            │
+                    │  • Namespace → Database mapping                            │
+                    │  • Topic → Table mapping                                  │
+                    │  • Schema version management                              │
+                    └─────────────────────────────────────────────────────────────┘
+                                                                        ▲
+                                                                        │ Metadata
+                                                                        │
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                          MQ Storage Layer                                      │
+│  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐    ▲    │
+│  │   Topic A   │  │   Topic B   │  │   Topic C   │  │     ...     │    │    │
+│  │ (Parquet)   │  │ (Parquet)   │  │ (Parquet)   │  │ (Parquet)   │    │    │
+│  └─────────────┘  └─────────────┘  └─────────────┘  └─────────────┘    │    │
+└──────────────────────────────────────────────────────────────────────────│──┘
+                                                                          │
+                                                                     Data Access
+```
+
+
+## Success Metrics
+
+*   **Feature Completeness:** Support for all specified SELECT operations and metadata commands
+*   **Performance:** 
+    *   **Simple SELECT queries**: < 100ms latency for single-table queries with up to 3 WHERE predicates on ≤ 100K records
+    *   **Complex queries**: < 1s latency for queries involving aggregations (COUNT, SUM, MAX, MIN) on ≤ 1M records
+    *   **Time-range queries**: < 500ms for timestamp-based filtering on ≤ 500K records within 24-hour windows
+*   **Scalability:** Handle topics with millions of messages efficiently

docker/Dockerfile.e2e

@@ -2,7 +2,18 @@ FROM ubuntu:22.04
 
 LABEL author="Chris Lu"
 
-RUN apt-get update && apt-get install -y curl fio fuse
+# Use faster mirrors and optimize package installation
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    --no-install-recommends \
+    --no-install-suggests \
+    curl \
+    fio \
+    fuse \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* \
+    && rm -rf /var/tmp/*
 RUN mkdir -p /etc/seaweedfs /data/filerldb2
 
 COPY ./weed /usr/bin/

docker/Dockerfile.rocksdb_dev_env

@@ -1,16 +1,17 @@
-FROM golang:1.24 as builder
+FROM golang:1.24 AS builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v10.2.1
+ARG ROCKSDB_VERSION=v10.5.1
+ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
 
 # build RocksDB
 RUN cd /tmp && \
     git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
     cd rocksdb && \
-    PORTABLE=1 make static_lib && \
+    PORTABLE=1 make -j"$(nproc)" static_lib && \
     make install-static
 
-ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
-ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+ENV CGO_CFLAGS="-I/tmp/rocksdb/include"
+ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"

docker/Dockerfile.rocksdb_large

@@ -1,24 +1,25 @@
-FROM golang:1.24 as builder
+FROM golang:1.24 AS builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v10.2.1
+ARG ROCKSDB_VERSION=v10.5.1
+ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
 
 # build RocksDB
 RUN cd /tmp && \
     git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
     cd rocksdb && \
-    PORTABLE=1 make static_lib && \
+    PORTABLE=1 make -j"$(nproc)" static_lib && \
     make install-static
 
-ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
-ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+ENV CGO_CFLAGS="-I/tmp/rocksdb/include"
+ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
 
 # build SeaweedFS
 RUN mkdir -p /go/src/github.com/seaweedfs/
 RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
-ARG BRANCH=${BRANCH:-master}
+ARG BRANCH=master
 RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \

docker/Makefile

@@ -20,7 +20,15 @@ build: binary
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 
 build_e2e: binary_race
-	docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
+	docker buildx build \
+		--cache-from=type=local,src=/tmp/.buildx-cache \
+		--cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \
+		--load \
+		-t chrislusf/seaweedfs:e2e \
+		-f Dockerfile.e2e .
+	# Move cache to avoid growing cache size
+	rm -rf /tmp/.buildx-cache || true
+	mv /tmp/.buildx-cache-new /tmp/.buildx-cache || true
 
 go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset,tarantool
 	docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .

docker/compose/e2e-mount.yml

@@ -6,16 +6,20 @@ services:
     command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
     healthcheck:
       test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
-      interval: 1s
-      timeout: 60s
+      interval: 2s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
 
   volume:
     image: chrislusf/seaweedfs:e2e
     command: "-v=4 volume -mserver=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
     healthcheck:
       test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz" ]
-      interval: 1s
-      timeout: 30s
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
     depends_on:
       master:
         condition: service_healthy
@@ -25,8 +29,10 @@ services:
     command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
     healthcheck:
       test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8888" ]
-      interval: 1s
-      timeout: 30s
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
     depends_on:
       volume:
         condition: service_healthy
@@ -46,8 +52,10 @@ services:
           memory: 4096m
     healthcheck:
       test: [ "CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs" ]
-      interval: 1s
-      timeout: 30s
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 10s
     depends_on:
       filer:
         condition: service_healthy

docker/compose/swarm-etcd.yml

@@ -1,6 +1,4 @@
 # 2021-01-30 16:25:30
-version: '3.8'
-
 services:
 
   etcd:

go.mod

@@ -1,15 +1,13 @@
 module github.com/seaweedfs/seaweedfs
 
-go 1.24
+go 1.24.0
 
 toolchain go1.24.1
 
 require (
 	cloud.google.com/go v0.121.6 // indirect
-	cloud.google.com/go/pubsub v1.50.0
-	cloud.google.com/go/storage v1.56.1
-	github.com/Azure/azure-pipeline-go v0.2.3
-	github.com/Azure/azure-storage-blob-go v0.15.0
+	cloud.google.com/go/pubsub v1.50.1
+	cloud.google.com/go/storage v1.57.0
 	github.com/Shopify/sarama v1.38.1
 	github.com/aws/aws-sdk-go v1.55.8
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -21,8 +19,8 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1
-	github.com/eapache/go-resiliency v1.3.0 // indirect
-	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
+	github.com/eapache/go-resiliency v1.6.0 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
@@ -30,12 +28,12 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-redsync/redsync/v4 v4.13.0
+	github.com/go-redsync/redsync/v4 v4.14.0
 	github.com/go-sql-driver/mysql v1.9.3
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.7.0
 	github.com/golang/protobuf v1.5.4
-	github.com/golang/snappy v1.0.0 // indirect
+	github.com/golang/snappy v1.0.0
 	github.com/google/btree v1.1.3
 	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.6.0 // indirect
@@ -45,19 +43,18 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/jackc/pgx/v5 v5.7.5
+	github.com/jackc/pgx/v5 v5.7.6
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jinzhu/copier v0.4.0
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.1
 	github.com/klauspost/reedsolomon v1.12.5
 	github.com/kurin/blazer v0.5.3
 	github.com/linxGnu/grocksdb v1.10.2
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-ieproxy v0.0.11 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -67,23 +64,23 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.23.0
+	github.com/prometheus/client_golang v1.23.2
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.17.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
 	github.com/seaweedfs/raft v1.1.3
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/spf13/afero v1.12.0 // indirect
-	github.com/spf13/cast v1.7.1 // indirect
-	github.com/spf13/viper v1.20.1
-	github.com/stretchr/testify v1.11.0
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
+	github.com/spf13/viper v1.21.0
+	github.com/stretchr/testify v1.11.1
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
 	github.com/tidwall/gjson v1.18.0
-	github.com/tidwall/match v1.1.1
+	github.com/tidwall/match v1.2.0
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
@@ -93,73 +90,80 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.4
+	go.etcd.io/etcd/client/v3 v3.6.5
 	go.mongodb.org/mongo-driver v1.17.4
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.43.0
 	gocloud.dev/pubsub/natspubsub v0.43.0
 	gocloud.dev/pubsub/rabbitpubsub v0.43.0
-	golang.org/x/crypto v0.41.0
-	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b
-	golang.org/x/image v0.30.0
-	golang.org/x/net v0.43.0
+	golang.org/x/crypto v0.43.0
+	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
+	golang.org/x/image v0.32.0
+	golang.org/x/net v0.45.0
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sys v0.35.0
-	golang.org/x/text v0.28.0 // indirect
-	golang.org/x/tools v0.36.0
+	golang.org/x/sys v0.37.0
+	golang.org/x/text v0.30.0 // indirect
+	golang.org/x/tools v0.37.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.247.0
 	google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
-	google.golang.org/grpc v1.75.0
-	google.golang.org/protobuf v1.36.8
+	google.golang.org/grpc v1.75.1
+	google.golang.org/protobuf v1.36.9
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/mathutil v1.7.1
 	modernc.org/memory v1.11.0 // indirect
-	modernc.org/sqlite v1.38.2
+	modernc.org/sqlite v1.39.0
 	modernc.org/strutil v1.2.1
 )
 
 require (
-	cloud.google.com/go/kms v1.22.0
+	cloud.google.com/go/kms v1.23.1
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/Jille/raft-grpc-transport v1.6.1
-	github.com/ThreeDotsLabs/watermill v1.5.0
-	github.com/a-h/templ v0.3.924
-	github.com/arangodb/go-driver v1.6.6
+	github.com/ThreeDotsLabs/watermill v1.5.1
+	github.com/a-h/templ v0.3.943
+	github.com/arangodb/go-driver v1.6.7
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.38.1
+	github.com/aws/aws-sdk-go-v2 v1.39.2
 	github.com/aws/aws-sdk-go-v2/config v1.31.3
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.7
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.10
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3
+	github.com/cockroachdb/cockroachdb-parser v0.25.3-0.20250917200107-7bd0408a9e14
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.35.0
+	github.com/getsentry/sentry-go v0.35.3
 	github.com/gin-contrib/sessions v1.0.4
-	github.com/gin-gonic/gin v1.10.1
+	github.com/gin-gonic/gin v1.11.0
 	github.com/golang-jwt/jwt/v5 v5.3.0
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hanwen/go-fuse/v2 v2.8.0
 	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
 	github.com/hashicorp/vault/api v1.20.0
+	github.com/jhump/protoreflect v1.17.0
+	github.com/lib/pq v1.10.9
+	github.com/linkedin/goavro/v2 v2.14.0
+	github.com/mattn/go-sqlite3 v1.14.32
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.25.1
 	github.com/pkg/sftp v1.13.9
 	github.com/rabbitmq/amqp091-go v1.10.0
-	github.com/rclone/rclone v1.70.3
+	github.com/rclone/rclone v1.71.1
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.12.1
+	github.com/redis/go-redis/v9 v9.14.1
 	github.com/schollz/progressbar/v3 v3.18.0
-	github.com/shirou/gopsutil/v3 v3.24.5
+	github.com/shirou/gopsutil/v4 v4.25.9
 	github.com/tarantool/go-tarantool/v2 v2.4.0
 	github.com/tikv/client-go/v2 v2.0.7
+	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.113.5
-	go.etcd.io/etcd/client/pkg/v3 v3.6.4
+	go.etcd.io/etcd/client/pkg/v3 v3.6.5
 	go.uber.org/atomic v1.11.0
-	golang.org/x/sync v0.16.0
+	golang.org/x/sync v0.17.0
+	golang.org/x/tools/godoc v0.1.0-deprecated
 	google.golang.org/grpc/security/advancedtls v1.0.0
 )
 
@@ -169,7 +173,21 @@ require (
 	cloud.google.com/go/longrunning v0.6.7 // indirect
 	cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
-	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
+	github.com/bazelbuild/rules_go v0.46.0 // indirect
+	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
+	github.com/blevesearch/snowballstem v0.9.0 // indirect
+	github.com/bufbuild/protocompile v0.14.1 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cockroachdb/apd/v3 v3.1.0 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect
+	github.com/dave/dst v0.27.2 // indirect
+	github.com/goccy/go-yaml v1.18.0 // indirect
+	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
@@ -178,8 +196,33 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/jaegertracing/jaeger v1.47.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
+	github.com/openzipkin/zipkin-go v0.4.3 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pierrre/geohash v1.0.0 // indirect
+	github.com/quic-go/qpack v0.5.1 // indirect
+	github.com/quic-go/quic-go v0.54.1 // indirect
+	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/sasha-s/go-deadlock v0.3.1 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/twpayne/go-geom v1.4.1 // indirect
+	github.com/twpayne/go-kml v1.5.2 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
+	go.uber.org/mock v0.5.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/mod v0.28.0 // indirect
+	gonum.org/v1/gonum v0.16.0 // indirect
 )
 
 require (
@@ -190,18 +233,18 @@ require (
 	cloud.google.com/go/iam v1.5.2 // indirect
 	cloud.google.com/go/monitoring v1.24.2 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.2
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.11.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
-	github.com/Files-com/files-sdk-go/v3 v3.2.173 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
+	github.com/Files-com/files-sdk-go/v3 v3.2.218 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
-	github.com/IBM/go-sdk-core/v5 v5.20.0 // indirect
+	github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
@@ -212,51 +255,51 @@ require (
 	github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
 	github.com/PuerkitoBio/goquery v1.10.3 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
-	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect
-	github.com/aws/smithy-go v1.22.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect
+	github.com/aws/smithy-go v1.23.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
-	github.com/bytedance/sonic v1.13.2 // indirect
-	github.com/bytedance/sonic/loader v0.2.4 // indirect
+	github.com/bytedance/sonic v1.14.0 // indirect
+	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
 	github.com/cloudflare/circl v1.6.1 // indirect
-	github.com/cloudinary/cloudinary-go/v2 v2.10.0 // indirect
+	github.com/cloudinary/cloudinary-go/v2 v2.12.0 // indirect
 	github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
 	github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
-	github.com/cloudwego/base64x v0.1.5 // indirect
+	github.com/cloudwego/base64x v0.1.6 // indirect
 	github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
 	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
 	github.com/creasty/defaults v1.8.0 // indirect
 	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
-	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
-	github.com/ebitengine/purego v0.8.4 // indirect
-	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/ebitengine/purego v0.9.0 // indirect
+	github.com/elastic/gosigar v0.14.3 // indirect
 	github.com/emersion/go-message v0.18.2 // indirect
 	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
@@ -266,18 +309,18 @@ require (
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
 	github.com/geoffgarside/ber v1.2.0 // indirect
-	github.com/gin-contrib/sse v1.0.0 // indirect
+	github.com/gin-contrib/sse v1.1.0 // indirect
 	github.com/go-chi/chi/v5 v5.2.2 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.1 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-openapi/errors v0.22.1 // indirect
+	github.com/go-openapi/errors v0.22.2 // indirect
 	github.com/go-openapi/strfmt v0.23.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.26.0 // indirect
+	github.com/go-playground/validator/v10 v10.27.0 // indirect
 	github.com/go-resty/resty/v2 v2.16.5 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
@@ -290,14 +333,14 @@ require (
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/sessions v1.4.0 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-metrics v0.5.4 // indirect
 	github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
 	github.com/henrybear327/go-proton-api v1.0.0 // indirect
@@ -311,12 +354,12 @@ require (
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
 	github.com/k0kubun/pp v3.0.1+incompatible
-	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lanrat/extsort v1.0.2 // indirect
+	github.com/lanrat/extsort v1.4.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lpar/date v1.0.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
@@ -324,7 +367,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nats-io/nats.go v1.43.0 // indirect
@@ -336,19 +379,19 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/onsi/ginkgo/v2 v2.23.3 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.93.0 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.98.0 // indirect
 	github.com/panjf2000/ants/v2 v2.11.3 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/xattr v0.4.10 // indirect
+	github.com/pkg/xattr v0.4.12 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
@@ -356,16 +399,14 @@ require (
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
-	github.com/sagikazarmark/locafero v0.7.0 // indirect
-	github.com/samber/lo v1.50.0 // indirect
-	github.com/shirou/gopsutil/v4 v4.25.5 // indirect
-	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sagikazarmark/locafero v0.11.0 // indirect
+	github.com/samber/lo v1.51.0 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/sony/gobreaker v1.0.0 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
-	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect
@@ -377,7 +418,7 @@ require (
 	github.com/tklauser/numcpus v0.10.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
-	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/ugorji/go/codec v1.3.0 // indirect
 	github.com/unknwon/goconfig v1.0.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
@@ -390,8 +431,8 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
-	go.etcd.io/bbolt v1.4.0 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.4 // indirect
+	go.etcd.io/bbolt v1.4.2 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.5 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.37.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect
@@ -403,8 +444,8 @@ require (
 	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/arch v0.16.0 // indirect
-	golang.org/x/term v0.34.0 // indirect
+	golang.org/x/arch v0.20.0 // indirect
+	golang.org/x/term v0.36.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
@@ -414,8 +455,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.66.3 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
-	storj.io/common v0.0.0-20250605163628-70ca83b6228e // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+	storj.io/common v0.0.0-20250808122759-804533d519c1 // indirect
 	storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
 	storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
 	storj.io/infectious v0.0.2 // indirect
@@ -423,4 +464,7 @@ require (
 	storj.io/uplink v1.13.1 // indirect
 )
 
+// Use the seaweedfs fork of cockroachdb-parser to fix cross-platform build issues
+replace github.com/cockroachdb/cockroachdb-parser => github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021182748-d0c58c67297e
+
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.97"
+appVersion: "3.98"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.397
+version: 4.0.398

k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml

@@ -79,6 +79,12 @@ spec:
           image: {{ template "master.image" . }}
           imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
           env:
+            {{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
+            {{- $envMerged := merge (.Values.global.extraEnvironmentVars | default dict) (.Values.allInOne.extraEnvironmentVars | default dict) }}
+            {{- $clusterDefault := default "sw" (index $envMerged "WEED_CLUSTER_DEFAULT") }}
+            {{- $clusterUpper := upper $clusterDefault }}
+            {{- $clusterMasterKey := printf "WEED_CLUSTER_%s_MASTER" $clusterUpper }}
+            {{- $clusterFilerKey := printf "WEED_CLUSTER_%s_FILER" $clusterUpper }}
             - name: POD_IP
               valueFrom:
                 fieldRef:
@@ -95,6 +101,7 @@ spec:
               value: "{{ template "seaweedfs.name" . }}"
             {{- if .Values.allInOne.extraEnvironmentVars }}
             {{- range $key, $value := .Values.allInOne.extraEnvironmentVars }}
+            {{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
             - name: {{ $key }}
               {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
@@ -104,8 +111,10 @@ spec:
               {{- end }}
             {{- end }}
             {{- end }}
+            {{- end }}
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
+            {{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
             - name: {{ $key }}
               {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
@@ -115,6 +124,12 @@ spec:
               {{- end }}
             {{- end }}
             {{- end }}
+            {{- end }}
+            # Inject computed cluster endpoints for the default cluster
+            - name: {{ $clusterMasterKey }}
+              value: {{ include "seaweedfs.cluster.masterAddress" . | quote }}
+            - name: {{ $clusterFilerKey }}
+              value: {{ include "seaweedfs.cluster.filerAddress" . | quote }}
           command:
             - "/bin/sh"
             - "-ec"

k8s/charts/seaweedfs/templates/cosi/cosi-deployment.yaml

@@ -15,7 +15,6 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: objectstorage-provisioner
   template:

k8s/charts/seaweedfs/templates/filer/filer-ingress.yaml

@@ -28,8 +28,8 @@ spec:
   rules:
   - http:
       paths:
-      - path: /sw-filer/?(.*)
-        pathType: ImplementationSpecific
+      - path: {{ .Values.filer.ingress.path | quote }}
+        pathType: {{ .Values.filer.ingress.pathType | quote }}
         backend:
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
           service:

k8s/charts/seaweedfs/templates/master/master-ingress.yaml

@@ -28,8 +28,8 @@ spec:
   rules:
     - http:
         paths:
-          - path: /sw-master/?(.*)
-            pathType: ImplementationSpecific
+          - path: {{ .Values.master.ingress.path | quote }}
+            pathType: {{ .Values.master.ingress.pathType | quote }}
             backend:
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
               service:

k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml

@@ -152,7 +152,10 @@ spec:
               {{- if .Values.s3.auditLogConfig }}
               -auditLogConfig=/etc/sw/s3_auditLogConfig.json \
               {{- end }}
-              -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
+              -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }} \
+              {{- range .Values.s3.extraArgs }}
+              {{ . }} \
+              {{- end }}
           volumeMounts:
             {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
             - name: logs

k8s/charts/seaweedfs/templates/s3/s3-ingress.yaml

@@ -27,8 +27,8 @@ spec:
   rules:
   - http:
       paths:
-      - path: /
-        pathType: ImplementationSpecific
+      - path: {{ .Values.s3.ingress.path | quote }}
+        pathType: {{ .Values.s3.ingress.pathType | quote }}
         backend:
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
           service:

k8s/charts/seaweedfs/templates/shared/_helpers.tpl

@@ -96,13 +96,16 @@ Inject extra environment vars in the format key:value, if populated
 {{/* Computes the container image name for all components (if they are not overridden) */}}
 {{- define "common.image" -}}
 {{- $registryName := default .Values.image.registry .Values.global.registry | toString -}}
-{{- $repositoryName := .Values.image.repository | toString -}}
+{{- $repositoryName := default .Values.image.repository .Values.global.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
 {{- $tag := default .Chart.AppVersion .Values.image.tag  | toString -}}
+{{- if $repositoryName -}}
+{{-   $name = printf "%s/%s" (trimSuffix "/" $repositoryName) (base $name) -}}
+{{- end -}}
 {{- if $registryName -}}
-{{- printf "%s/%s%s:%s" $registryName $repositoryName $name $tag -}}
+{{-   printf "%s/%s:%s" $registryName $name $tag -}}
 {{- else -}}
-{{- printf "%s%s:%s" $repositoryName $name $tag -}}
+{{-   printf "%s:%s" $name $tag -}}
 {{- end -}}
 {{- end -}}
 
@@ -219,3 +222,27 @@ or generate a new random password if it doesn't exist.
   {{- randAlphaNum $length -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Compute the master service address to be used in cluster env vars.
+If allInOne is enabled, point to the all-in-one service; otherwise, point to the master service.
+*/}}
+{{- define "seaweedfs.cluster.masterAddress" -}}
+{{- $serviceNameSuffix := "-master" -}}
+{{- if .Values.allInOne.enabled -}}
+{{-   $serviceNameSuffix = "-all-in-one" -}}
+{{- end -}}
+{{- printf "%s%s.%s:%d" (include "seaweedfs.name" .) $serviceNameSuffix .Release.Namespace (int .Values.master.port) -}}
+{{- end -}}
+
+{{/*
+Compute the filer service address to be used in cluster env vars.
+If allInOne is enabled, point to the all-in-one service; otherwise, point to the filer-client service.
+*/}}
+{{- define "seaweedfs.cluster.filerAddress" -}}
+{{- $serviceNameSuffix := "-filer-client" -}}
+{{- if .Values.allInOne.enabled -}}
+{{-   $serviceNameSuffix = "-all-in-one" -}}
+{{- end -}}
+{{- printf "%s%s.%s:%d" (include "seaweedfs.name" .) $serviceNameSuffix .Release.Namespace (int .Values.filer.port) -}}
+{{- end -}}

k8s/charts/seaweedfs/templates/volume/volume-servicemonitor.yaml

@@ -21,9 +21,9 @@ metadata:
     {{- with $.Values.global.monitoring.additionalLabels }}
     {{- toYaml . | nindent 4 }}
     {{- end }}
-{{- if .Values.volume.annotations }}
+{{- with $volume.annotations }}
   annotations:
-    {{- toYaml .Values.volume.annotations | nindent 4 }}
+    {{- toYaml . | nindent 4 }}
 {{- end }}
 spec:
   endpoints:

k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml

@@ -88,6 +88,9 @@ spec:
             - name: {{ $dir.name }}
               mountPath: /{{ $dir.name }}
           {{- end }}
+          {{- if $volume.containerSecurityContext.enabled }}
+          securityContext: {{- omit $volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
         {{- end }}
         {{- if $volume.initContainers }}
         {{ tpl (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.initContainers) $ | indent 8 | trim }}

k8s/charts/seaweedfs/values.yaml

@@ -3,6 +3,7 @@
 global:
   createClusterRole: true
   registry: ""
+  # if repository is set, it overrides the namespace part of imageName
   repository: ""
   imageName: chrislusf/seaweedfs
   imagePullPolicy: IfNotPresent
@@ -201,8 +202,7 @@ master:
   # nodeSelector labels for master pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-    kubernetes.io/arch: amd64
+  nodeSelector: ""
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -238,6 +238,8 @@ master:
     className: "nginx"
     # host: false for "*" hostname
     host: "master.seaweedfs.local"
+    path: "/sw-master/?(.*)"
+    pathType: ImplementationSpecific
     annotations:
       nginx.ingress.kubernetes.io/auth-type: "basic"
       nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
@@ -478,8 +480,7 @@ volume:
   # nodeSelector labels for server pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-   kubernetes.io/arch: amd64
+  nodeSelector: ""
   # nodeSelector: |
   #   sw-volume: "true"
 
@@ -735,8 +736,7 @@ filer:
   # nodeSelector labels for server pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-    kubernetes.io/arch: amd64
+  nodeSelector: ""
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -772,6 +772,8 @@ filer:
     className: "nginx"
     # host: false for "*" hostname
     host: "seaweedfs.cluster.local"
+    path: "/sw-filer/?(.*)"
+    pathType: ImplementationSpecific
     annotations:
       nginx.ingress.kubernetes.io/backend-protocol: GRPC
       nginx.ingress.kubernetes.io/auth-type: "basic"
@@ -871,7 +873,7 @@ filer:
     #     anonymousRead: false
 
 s3:
-  enabled: false
+  enabled: true
   imageOverride: null
   restartPolicy: null
   replicas: 1
@@ -932,8 +934,7 @@ s3:
   # nodeSelector labels for server pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
-  nodeSelector: |
-    kubernetes.io/arch: amd64
+  nodeSelector: ""
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -975,6 +976,11 @@ s3:
 
   extraEnvironmentVars:
 
+  # Custom command line arguments to add to the s3 command
+  # Example to fix connection idle seconds:
+  extraArgs: ["-idleTimeout=30"]
+  #extraArgs: []
+
   # used to configure livenessProbe on s3 containers
   #
   livenessProbe:
@@ -1006,6 +1012,8 @@ s3:
     className: "nginx"
     # host: false for "*" hostname
     host: "seaweedfs.cluster.local"
+    path: "/"
+    pathType: Prefix
     # additional ingress annotations for the s3 endpoint
     annotations: {}
     tls: []
@@ -1051,8 +1059,7 @@ sftp:
   annotations: {}
   resources: {}
   tolerations: ""
-  nodeSelector: |
-    kubernetes.io/arch: amd64
+  nodeSelector: ""
   priorityClassName: ""
   serviceAccountName: ""
   podSecurityContext: {}
@@ -1179,8 +1186,7 @@ allInOne:
 
   # nodeSelector labels for master pod assignment, formatted as a muli-line string.
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
-  nodeSelector: |
-    kubernetes.io/arch: amd64
+  nodeSelector: ""
 
   # Used to assign priority to master pods
   # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/

other/java/client/pom.xml

@@ -33,7 +33,7 @@
     <properties>
         <protobuf.version>3.25.5</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
-        <grpc.version>1.68.1</grpc.version>
+        <grpc.version>1.75.0</grpc.version>
         <guava.version>32.0.0-jre</guava.version>
     </properties>
 

other/java/client/src/main/proto/filer.proto

@@ -162,7 +162,7 @@ message FileChunk {
     bool is_compressed = 10;
     bool is_chunk_manifest = 11; // content is a list of FileChunks
     SSEType sse_type = 12;           // Server-side encryption type
-    bytes sse_kms_metadata = 13;     // Serialized SSE-KMS metadata for this chunk
+    bytes sse_metadata = 13;         // Serialized SSE metadata for this chunk (SSE-C, SSE-KMS, or SSE-S3)
 }
 
 message FileChunkManifest {
@@ -390,6 +390,7 @@ message LogEntry {
     int32 partition_key_hash = 2;
     bytes data = 3;
     bytes key = 4;
+    int64 offset = 5;  // Sequential offset within partition
 }
 
 message KeepConnectedRequest {

postgres-examples/README.md

@@ -0,0 +1,414 @@
+# SeaweedFS PostgreSQL Protocol Examples
+
+This directory contains examples demonstrating how to connect to SeaweedFS using the PostgreSQL wire protocol.
+
+## Starting the PostgreSQL Server
+
+```bash
+# Start with trust authentication (no password required)
+weed postgres -port=5432 -master=localhost:9333
+
+# Start with password authentication
+weed postgres -port=5432 -auth=password -users="admin:secret;readonly:view123"
+
+# Start with MD5 authentication (more secure)
+weed postgres -port=5432 -auth=md5 -users="user1:pass1;user2:pass2"
+
+# Start with TLS encryption
+weed postgres -port=5432 -tls-cert=server.crt -tls-key=server.key
+
+# Allow connections from any host
+weed postgres -host=0.0.0.0 -port=5432
+```
+
+## Client Connections
+
+### psql Command Line
+
+```bash
+# Basic connection (trust auth)
+psql -h localhost -p 5432 -U seaweedfs -d default
+
+# With password
+PGPASSWORD=secret psql -h localhost -p 5432 -U admin -d default
+
+# Connection string format
+psql "postgresql://admin:secret@localhost:5432/default"
+
+# Connection string with parameters
+psql "host=localhost port=5432 dbname=default user=admin password=secret"
+```
+
+### Programming Languages
+
+#### Python (psycopg2)
+```python
+import psycopg2
+
+# Connect to SeaweedFS
+conn = psycopg2.connect(
+    host="localhost",
+    port=5432,
+    user="seaweedfs", 
+    database="default"
+)
+
+# Execute queries
+cursor = conn.cursor()
+cursor.execute("SELECT * FROM my_topic LIMIT 10")
+
+for row in cursor.fetchall():
+    print(row)
+
+cursor.close()
+conn.close()
+```
+
+#### Java JDBC
+```java
+import java.sql.*;
+
+public class SeaweedFSExample {
+    public static void main(String[] args) throws SQLException {
+        String url = "jdbc:postgresql://localhost:5432/default";
+        
+        Connection conn = DriverManager.getConnection(url, "seaweedfs", "");
+        Statement stmt = conn.createStatement();
+        
+        ResultSet rs = stmt.executeQuery("SELECT * FROM my_topic LIMIT 10");
+        while (rs.next()) {
+            System.out.println("ID: " + rs.getLong("id"));
+            System.out.println("Message: " + rs.getString("message"));
+        }
+        
+        rs.close();
+        stmt.close(); 
+        conn.close();
+    }
+}
+```
+
+#### Go (lib/pq)
+```go
+package main
+
+import (
+    "database/sql"
+    "fmt"
+    _ "github.com/lib/pq"
+)
+
+func main() {
+    db, err := sql.Open("postgres", 
+        "host=localhost port=5432 user=seaweedfs dbname=default sslmode=disable")
+    if err != nil {
+        panic(err)
+    }
+    defer db.Close()
+
+    rows, err := db.Query("SELECT * FROM my_topic LIMIT 10")
+    if err != nil {
+        panic(err)
+    }
+    defer rows.Close()
+
+    for rows.Next() {
+        var id int64
+        var message string
+        err := rows.Scan(&id, &message)
+        if err != nil {
+            panic(err)
+        }
+        fmt.Printf("ID: %d, Message: %s\n", id, message)
+    }
+}
+```
+
+#### Node.js (pg)
+```javascript
+const { Client } = require('pg');
+
+const client = new Client({
+    host: 'localhost',
+    port: 5432,
+    user: 'seaweedfs',
+    database: 'default',
+});
+
+async function query() {
+    await client.connect();
+    
+    const result = await client.query('SELECT * FROM my_topic LIMIT 10');
+    console.log(result.rows);
+    
+    await client.end();
+}
+
+query().catch(console.error);
+```
+
+## SQL Operations
+
+### Basic Queries
+```sql
+-- List databases
+SHOW DATABASES;
+
+-- List tables (topics)
+SHOW TABLES;
+
+-- Describe table structure
+DESCRIBE my_topic;
+-- or use the shorthand: DESC my_topic;
+
+-- Basic select
+SELECT * FROM my_topic;
+
+-- With WHERE clause
+SELECT id, message FROM my_topic WHERE id > 1000;
+
+-- With LIMIT
+SELECT * FROM my_topic LIMIT 100;
+```
+
+### Aggregations
+```sql
+-- Count records
+SELECT COUNT(*) FROM my_topic;
+
+-- Multiple aggregations
+SELECT 
+    COUNT(*) as total_messages,
+    MIN(id) as min_id,
+    MAX(id) as max_id,
+    AVG(amount) as avg_amount
+FROM my_topic;
+
+-- Aggregations with WHERE
+SELECT COUNT(*) FROM my_topic WHERE status = 'active';
+```
+
+### System Columns
+```sql
+-- Access system columns
+SELECT 
+    id,
+    message, 
+    _timestamp_ns as timestamp,
+    _key as partition_key,
+    _source as data_source
+FROM my_topic;
+
+-- Filter by timestamp
+SELECT * FROM my_topic 
+WHERE _timestamp_ns > 1640995200000000000
+LIMIT 10;
+```
+
+### PostgreSQL System Queries
+```sql
+-- Version information
+SELECT version();
+
+-- Current database
+SELECT current_database();
+
+-- Current user
+SELECT current_user;
+
+-- Server settings
+SELECT current_setting('server_version');
+SELECT current_setting('server_encoding');
+```
+
+## psql Meta-Commands
+
+```sql
+-- List tables
+\d
+\dt
+
+-- List databases  
+\l
+
+-- Describe specific table
+\d my_topic
+\dt my_topic
+
+-- List schemas
+\dn
+
+-- Help
+\h
+\?
+
+-- Quit
+\q
+```
+
+## Database Tools Integration
+
+### DBeaver
+1. Create New Connection → PostgreSQL
+2. Settings:
+   - **Host**: localhost
+   - **Port**: 5432
+   - **Database**: default
+   - **Username**: seaweedfs (or configured user)
+   - **Password**: (if using password auth)
+
+### pgAdmin
+1. Add New Server
+2. Connection tab:
+   - **Host**: localhost
+   - **Port**: 5432
+   - **Username**: seaweedfs
+   - **Database**: default
+
+### DataGrip
+1. New Data Source → PostgreSQL
+2. Configure:
+   - **Host**: localhost
+   - **Port**: 5432
+   - **User**: seaweedfs
+   - **Database**: default
+
+### Grafana
+1. Add Data Source → PostgreSQL
+2. Configuration:
+   - **Host**: localhost:5432
+   - **Database**: default
+   - **User**: seaweedfs
+   - **SSL Mode**: disable
+
+## BI Tools
+
+### Tableau
+1. Connect to Data → PostgreSQL
+2. Server: localhost
+3. Port: 5432
+4. Database: default
+5. Username: seaweedfs
+
+### Power BI
+1. Get Data → Database → PostgreSQL
+2. Server: localhost
+3. Database: default
+4. Username: seaweedfs
+
+## Connection Pooling
+
+### Java (HikariCP)
+```java
+HikariConfig config = new HikariConfig();
+config.setJdbcUrl("jdbc:postgresql://localhost:5432/default");
+config.setUsername("seaweedfs");
+config.setMaximumPoolSize(10);
+
+HikariDataSource dataSource = new HikariDataSource(config);
+```
+
+### Python (connection pooling)
+```python
+from psycopg2 import pool
+
+connection_pool = psycopg2.pool.SimpleConnectionPool(
+    1, 20,
+    host="localhost",
+    port=5432,
+    user="seaweedfs",
+    database="default"
+)
+
+conn = connection_pool.getconn()
+# Use connection
+connection_pool.putconn(conn)
+```
+
+## Security Best Practices
+
+### Use TLS Encryption
+```bash
+# Generate self-signed certificate for testing
+openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 365 -nodes
+
+# Start with TLS
+weed postgres -tls-cert=server.crt -tls-key=server.key
+```
+
+### Use MD5 Authentication
+```bash
+# More secure than password auth
+weed postgres -auth=md5 -users="admin:secret123;readonly:view456"
+```
+
+### Limit Connections
+```bash
+# Limit concurrent connections
+weed postgres -max-connections=50 -idle-timeout=30m
+```
+
+## Troubleshooting
+
+### Connection Issues
+```bash
+# Test connectivity
+telnet localhost 5432
+
+# Check if server is running
+ps aux | grep "weed postgres"
+
+# Check logs for errors
+tail -f /var/log/seaweedfs/postgres.log
+```
+
+### Common Errors
+
+**"Connection refused"**
+- Ensure PostgreSQL server is running
+- Check host/port configuration
+- Verify firewall settings
+
+**"Authentication failed"**
+- Check username/password
+- Verify auth method configuration
+- Ensure user is configured in server
+
+**"Database does not exist"**
+- Use correct database name (default: 'default')
+- Check available databases: `SHOW DATABASES`
+
+**"Permission denied"**
+- Check user permissions
+- Verify authentication method
+- Use correct credentials
+
+## Performance Tips
+
+1. **Use LIMIT clauses** for large result sets
+2. **Filter with WHERE clauses** to reduce data transfer
+3. **Use connection pooling** for multi-threaded applications
+4. **Close resources properly** (connections, statements, result sets)
+5. **Use prepared statements** for repeated queries
+
+## Monitoring
+
+### Connection Statistics
+```sql
+-- Current connections (if supported)
+SELECT COUNT(*) FROM pg_stat_activity;
+
+-- Server version
+SELECT version();
+
+-- Current settings
+SELECT name, setting FROM pg_settings WHERE name LIKE '%connection%';
+```
+
+### Query Performance
+```sql
+-- Use EXPLAIN for query plans (if supported)
+EXPLAIN SELECT * FROM my_topic WHERE id > 1000;
+```
+
+This PostgreSQL protocol support makes SeaweedFS accessible to the entire PostgreSQL ecosystem, enabling seamless integration with existing tools, applications, and workflows.

postgres-examples/test_client.py

@@ -0,0 +1,374 @@
+#!/usr/bin/env python3
+"""
+Test client for SeaweedFS PostgreSQL protocol support.
+
+This script demonstrates how to connect to SeaweedFS using standard PostgreSQL
+libraries and execute various types of queries.
+
+Requirements:
+    pip install psycopg2-binary
+
+Usage:
+    python test_client.py
+    python test_client.py --host localhost --port 5432 --user seaweedfs --database default
+"""
+
+import sys
+import argparse
+import time
+import traceback
+
+try:
+    import psycopg2
+    import psycopg2.extras
+except ImportError:
+    print("Error: psycopg2 not found. Install with: pip install psycopg2-binary")
+    sys.exit(1)
+
+
+def test_connection(host, port, user, database, password=None):
+    """Test basic connection to SeaweedFS PostgreSQL server."""
+    print(f"🔗 Testing connection to {host}:{port}/{database} as user '{user}'")
+    
+    try:
+        conn_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'database': database,
+            'connect_timeout': 10
+        }
+        
+        if password:
+            conn_params['password'] = password
+            
+        conn = psycopg2.connect(**conn_params)
+        print("✅ Connection successful!")
+        
+        # Test basic query
+        cursor = conn.cursor()
+        cursor.execute("SELECT 1 as test")
+        result = cursor.fetchone()
+        print(f"✅ Basic query successful: {result}")
+        
+        cursor.close()
+        conn.close()
+        return True
+        
+    except Exception as e:
+        print(f"❌ Connection failed: {e}")
+        return False
+
+
+def test_system_queries(host, port, user, database, password=None):
+    """Test PostgreSQL system queries."""
+    print("\n🔧 Testing PostgreSQL system queries...")
+    
+    try:
+        conn_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'database': database
+        }
+        if password:
+            conn_params['password'] = password
+            
+        conn = psycopg2.connect(**conn_params)
+        cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        
+        system_queries = [
+            ("Version", "SELECT version()"),
+            ("Current Database", "SELECT current_database()"),
+            ("Current User", "SELECT current_user"),
+            ("Server Encoding", "SELECT current_setting('server_encoding')"),
+            ("Client Encoding", "SELECT current_setting('client_encoding')"),
+        ]
+        
+        for name, query in system_queries:
+            try:
+                cursor.execute(query)
+                result = cursor.fetchone()
+                print(f"  ✅ {name}: {result[0]}")
+            except Exception as e:
+                print(f"  ❌ {name}: {e}")
+        
+        cursor.close()
+        conn.close()
+        
+    except Exception as e:
+        print(f"❌ System queries failed: {e}")
+
+
+def test_schema_queries(host, port, user, database, password=None):
+    """Test schema and metadata queries."""
+    print("\n📊 Testing schema queries...")
+    
+    try:
+        conn_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'database': database
+        }
+        if password:
+            conn_params['password'] = password
+            
+        conn = psycopg2.connect(**conn_params)
+        cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        
+        schema_queries = [
+            ("Show Databases", "SHOW DATABASES"),
+            ("Show Tables", "SHOW TABLES"),
+            ("List Schemas", "SELECT 'public' as schema_name"),
+        ]
+        
+        for name, query in schema_queries:
+            try:
+                cursor.execute(query)
+                results = cursor.fetchall()
+                print(f"  ✅ {name}: Found {len(results)} items")
+                for row in results[:3]:  # Show first 3 results
+                    print(f"    - {dict(row)}")
+                if len(results) > 3:
+                    print(f"    ... and {len(results) - 3} more")
+            except Exception as e:
+                print(f"  ❌ {name}: {e}")
+        
+        cursor.close()
+        conn.close()
+        
+    except Exception as e:
+        print(f"❌ Schema queries failed: {e}")
+
+
+def test_data_queries(host, port, user, database, password=None):
+    """Test data queries on actual topics."""
+    print("\n📝 Testing data queries...")
+    
+    try:
+        conn_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'database': database
+        }
+        if password:
+            conn_params['password'] = password
+            
+        conn = psycopg2.connect(**conn_params)
+        cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        
+        # First, try to get available tables/topics
+        cursor.execute("SHOW TABLES")
+        tables = cursor.fetchall()
+        
+        if not tables:
+            print("  ℹ️  No tables/topics found for data testing")
+            cursor.close()
+            conn.close()
+            return
+            
+        # Test with first available table
+        table_name = tables[0][0] if tables[0] else 'test_topic'
+        print(f"  📋 Testing with table: {table_name}")
+        
+        test_queries = [
+            (f"Count records in {table_name}", f"SELECT COUNT(*) FROM \"{table_name}\""),
+            (f"Sample data from {table_name}", f"SELECT * FROM \"{table_name}\" LIMIT 3"),
+            (f"System columns from {table_name}", f"SELECT _timestamp_ns, _key, _source FROM \"{table_name}\" LIMIT 3"),
+            (f"Describe {table_name}", f"DESCRIBE \"{table_name}\""),
+        ]
+        
+        for name, query in test_queries:
+            try:
+                cursor.execute(query)
+                results = cursor.fetchall()
+                
+                if "COUNT" in query.upper():
+                    count = results[0][0] if results else 0
+                    print(f"  ✅ {name}: {count} records")
+                elif "DESCRIBE" in query.upper():
+                    print(f"  ✅ {name}: {len(results)} columns")
+                    for row in results[:5]:  # Show first 5 columns
+                        print(f"    - {dict(row)}")
+                else:
+                    print(f"  ✅ {name}: {len(results)} rows")
+                    for row in results:
+                        print(f"    - {dict(row)}")
+                        
+            except Exception as e:
+                print(f"  ❌ {name}: {e}")
+        
+        cursor.close()
+        conn.close()
+        
+    except Exception as e:
+        print(f"❌ Data queries failed: {e}")
+
+
+def test_prepared_statements(host, port, user, database, password=None):
+    """Test prepared statements."""
+    print("\n📝 Testing prepared statements...")
+    
+    try:
+        conn_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'database': database
+        }
+        if password:
+            conn_params['password'] = password
+            
+        conn = psycopg2.connect(**conn_params)
+        cursor = conn.cursor()
+        
+        # Test parameterized query
+        try:
+            cursor.execute("SELECT %s as param1, %s as param2", ("hello", 42))
+            result = cursor.fetchone()
+            print(f"  ✅ Prepared statement: {result}")
+        except Exception as e:
+            print(f"  ❌ Prepared statement: {e}")
+        
+        cursor.close()
+        conn.close()
+        
+    except Exception as e:
+        print(f"❌ Prepared statements test failed: {e}")
+
+
+def test_transaction_support(host, port, user, database, password=None):
+    """Test transaction support (should be no-op for read-only)."""
+    print("\n🔄 Testing transaction support...")
+    
+    try:
+        conn_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'database': database
+        }
+        if password:
+            conn_params['password'] = password
+            
+        conn = psycopg2.connect(**conn_params)
+        cursor = conn.cursor()
+        
+        transaction_commands = [
+            "BEGIN",
+            "SELECT 1 as in_transaction", 
+            "COMMIT",
+            "SELECT 1 as after_commit",
+        ]
+        
+        for cmd in transaction_commands:
+            try:
+                cursor.execute(cmd)
+                if "SELECT" in cmd:
+                    result = cursor.fetchone()
+                    print(f"  ✅ {cmd}: {result}")
+                else:
+                    print(f"  ✅ {cmd}: OK")
+            except Exception as e:
+                print(f"  ❌ {cmd}: {e}")
+        
+        cursor.close()
+        conn.close()
+        
+    except Exception as e:
+        print(f"❌ Transaction test failed: {e}")
+
+
+def test_performance(host, port, user, database, password=None, iterations=10):
+    """Test query performance."""
+    print(f"\n⚡ Testing performance ({iterations} iterations)...")
+    
+    try:
+        conn_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'database': database
+        }
+        if password:
+            conn_params['password'] = password
+            
+        times = []
+        
+        for i in range(iterations):
+            start_time = time.time()
+            
+            conn = psycopg2.connect(**conn_params)
+            cursor = conn.cursor()
+            cursor.execute("SELECT 1")
+            result = cursor.fetchone()
+            cursor.close()
+            conn.close()
+            
+            elapsed = time.time() - start_time
+            times.append(elapsed)
+            
+            if i < 3:  # Show first 3 iterations
+                print(f"  Iteration {i+1}: {elapsed:.3f}s")
+        
+        avg_time = sum(times) / len(times)
+        min_time = min(times)
+        max_time = max(times)
+        
+        print(f"  ✅ Performance results:")
+        print(f"    - Average: {avg_time:.3f}s")
+        print(f"    - Min: {min_time:.3f}s") 
+        print(f"    - Max: {max_time:.3f}s")
+        
+    except Exception as e:
+        print(f"❌ Performance test failed: {e}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Test SeaweedFS PostgreSQL Protocol")
+    parser.add_argument("--host", default="localhost", help="PostgreSQL server host")
+    parser.add_argument("--port", type=int, default=5432, help="PostgreSQL server port")
+    parser.add_argument("--user", default="seaweedfs", help="PostgreSQL username")
+    parser.add_argument("--password", help="PostgreSQL password")
+    parser.add_argument("--database", default="default", help="PostgreSQL database")
+    parser.add_argument("--skip-performance", action="store_true", help="Skip performance tests")
+    
+    args = parser.parse_args()
+    
+    print("🧪 SeaweedFS PostgreSQL Protocol Test Client")
+    print("=" * 50)
+    
+    # Test basic connection first
+    if not test_connection(args.host, args.port, args.user, args.database, args.password):
+        print("\n❌ Basic connection failed. Cannot continue with other tests.")
+        sys.exit(1)
+    
+    # Run all tests
+    try:
+        test_system_queries(args.host, args.port, args.user, args.database, args.password)
+        test_schema_queries(args.host, args.port, args.user, args.database, args.password)
+        test_data_queries(args.host, args.port, args.user, args.database, args.password)
+        test_prepared_statements(args.host, args.port, args.user, args.database, args.password)
+        test_transaction_support(args.host, args.port, args.user, args.database, args.password)
+        
+        if not args.skip_performance:
+            test_performance(args.host, args.port, args.user, args.database, args.password)
+            
+    except KeyboardInterrupt:
+        print("\n\n⚠️  Tests interrupted by user")
+        sys.exit(0)
+    except Exception as e:
+        print(f"\n❌ Unexpected error during testing: {e}")
+        traceback.print_exc()
+        sys.exit(1)
+    
+    print("\n🎉 All tests completed!")
+    print("\nTo use SeaweedFS with PostgreSQL tools:")
+    print(f"  psql -h {args.host} -p {args.port} -U {args.user} -d {args.database}")
+    print(f"  Connection string: postgresql://{args.user}@{args.host}:{args.port}/{args.database}")
+
+
+if __name__ == "__main__":
+    main()

seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   # SeaweedFS Master
   seaweedfs-master:

seaweedfs-rdma-sidecar/test-fixes-standalone.go

@@ -31,7 +31,7 @@ func parseUint64(s string, defaultValue uint64) uint64 {
 
 // Test the improved error reporting pattern (from weed/mount/rdma_client.go fix)
 func testErrorReporting() {
-	fmt.Println("🔧 Testing Error Reporting Fix:")
+	fmt.Println("Testing Error Reporting Fix:")
 
 	// Simulate RDMA failure followed by HTTP failure
 	rdmaErr := fmt.Errorf("RDMA connection timeout")
@@ -39,24 +39,24 @@ func testErrorReporting() {
 
 	// OLD (incorrect) way:
 	oldError := fmt.Errorf("both RDMA and HTTP fallback failed: RDMA=%v, HTTP=%v", rdmaErr, rdmaErr) // BUG: same error twice
-	fmt.Printf("  ❌ Old (buggy): %v\n", oldError)
+	fmt.Printf("  Old (buggy): %v\n", oldError)
 
 	// NEW (fixed) way:
 	newError := fmt.Errorf("both RDMA and HTTP fallback failed: RDMA=%v, HTTP=%v", rdmaErr, httpErr) // FIXED: different errors
-	fmt.Printf("  ✅ New (fixed): %v\n", newError)
+	fmt.Printf("  New (fixed): %v\n", newError)
 }
 
 // Test weed mount command with RDMA flags (from docker-compose fix)
 func testWeedMountCommand() {
-	fmt.Println("🔧 Testing Weed Mount Command Fix:")
+	fmt.Println("Testing Weed Mount Command Fix:")
 
 	// OLD (missing RDMA flags):
 	oldCommand := "/usr/local/bin/weed mount -filer=seaweedfs-filer:8888 -dir=/mnt/seaweedfs -allowOthers=true -debug"
-	fmt.Printf("  ❌ Old (missing RDMA): %s\n", oldCommand)
+	fmt.Printf("  Old (missing RDMA): %s\n", oldCommand)
 
 	// NEW (with RDMA flags):
 	newCommand := "/usr/local/bin/weed mount -filer=${FILER_ADDR} -dir=${MOUNT_POINT} -allowOthers=true -rdma.enabled=${RDMA_ENABLED} -rdma.sidecar=${RDMA_SIDECAR_ADDR} -rdma.fallback=${RDMA_FALLBACK} -rdma.maxConcurrent=${RDMA_MAX_CONCURRENT} -rdma.timeoutMs=${RDMA_TIMEOUT_MS} -debug=${DEBUG}"
-	fmt.Printf("  ✅ New (with RDMA): %s\n", newCommand)
+	fmt.Printf("  New (with RDMA): %s\n", newCommand)
 
 	// Check if RDMA flags are present
 	rdmaFlags := []string{"-rdma.enabled", "-rdma.sidecar", "-rdma.fallback", "-rdma.maxConcurrent", "-rdma.timeoutMs"}
@@ -69,38 +69,38 @@ func testWeedMountCommand() {
 	}
 
 	if allPresent {
-		fmt.Println("  ✅ All RDMA flags present in command")
+		fmt.Println("  All RDMA flags present in command")
 	} else {
-		fmt.Println("  ❌ Missing RDMA flags")
+		fmt.Println("  Missing RDMA flags")
 	}
 }
 
 // Test health check robustness (from Dockerfile.rdma-engine fix)
 func testHealthCheck() {
-	fmt.Println("🔧 Testing Health Check Fix:")
+	fmt.Println("Testing Health Check Fix:")
 
 	// OLD (hardcoded):
 	oldHealthCheck := "test -S /tmp/rdma-engine.sock"
-	fmt.Printf("  ❌ Old (hardcoded): %s\n", oldHealthCheck)
+	fmt.Printf("  Old (hardcoded): %s\n", oldHealthCheck)
 
 	// NEW (robust):
 	newHealthCheck := `pgrep rdma-engine-server >/dev/null && test -d /tmp/rdma && test "$(find /tmp/rdma -name '*.sock' | wc -l)" -gt 0`
-	fmt.Printf("  ✅ New (robust): %s\n", newHealthCheck)
+	fmt.Printf("  New (robust): %s\n", newHealthCheck)
 }
 
 func main() {
-	fmt.Println("🎯 Testing All GitHub PR Review Fixes")
+	fmt.Println("Testing All GitHub PR Review Fixes")
 	fmt.Println("====================================")
 	fmt.Println()
 
 	// Test parse functions
-	fmt.Println("🔧 Testing Parse Functions Fix:")
+	fmt.Println("Testing Parse Functions Fix:")
 	fmt.Printf("  parseUint32('123', 0) = %d (expected: 123)\n", parseUint32("123", 0))
 	fmt.Printf("  parseUint32('', 999) = %d (expected: 999)\n", parseUint32("", 999))
 	fmt.Printf("  parseUint32('invalid', 999) = %d (expected: 999)\n", parseUint32("invalid", 999))
 	fmt.Printf("  parseUint64('12345678901234', 0) = %d (expected: 12345678901234)\n", parseUint64("12345678901234", 0))
 	fmt.Printf("  parseUint64('invalid', 999) = %d (expected: 999)\n", parseUint64("invalid", 999))
-	fmt.Println("  ✅ Parse functions handle errors correctly!")
+	fmt.Println("  Parse functions handle errors correctly!")
 	fmt.Println()
 
 	testErrorReporting()
@@ -112,16 +112,16 @@ func main() {
 	testHealthCheck()
 	fmt.Println()
 
-	fmt.Println("🎉 All Review Fixes Validated!")
+	fmt.Println("All Review Fixes Validated!")
 	fmt.Println("=============================")
 	fmt.Println()
-	fmt.Println("✅ Parse functions: Safe error handling with strconv.ParseUint")
-	fmt.Println("✅ Error reporting: Proper distinction between RDMA and HTTP errors")
-	fmt.Println("✅ Weed mount: RDMA flags properly included in Docker command")
-	fmt.Println("✅ Health check: Robust socket detection without hardcoding")
-	fmt.Println("✅ File ID parsing: Reuses existing SeaweedFS functions")
-	fmt.Println("✅ Semaphore handling: No more channel close panics")
-	fmt.Println("✅ Go.mod documentation: Clear instructions for contributors")
+	fmt.Println("Parse functions: Safe error handling with strconv.ParseUint")
+	fmt.Println("Error reporting: Proper distinction between RDMA and HTTP errors")
+	fmt.Println("Weed mount: RDMA flags properly included in Docker command")
+	fmt.Println("Health check: Robust socket detection without hardcoding")
+	fmt.Println("File ID parsing: Reuses existing SeaweedFS functions")
+	fmt.Println("Semaphore handling: No more channel close panics")
+	fmt.Println("Go.mod documentation: Clear instructions for contributors")
 	fmt.Println()
-	fmt.Println("🚀 Ready for production deployment!")
+	fmt.Println("Ready for production deployment!")
 }

telemetry/DEPLOYMENT.md

@@ -1,6 +1,6 @@
 # SeaweedFS Telemetry Server Deployment
 
-This document describes how to deploy the SeaweedFS telemetry server to a remote server using GitHub Actions.
+This document describes how to deploy the SeaweedFS telemetry server to a remote server using GitHub Actions, or via Docker.
 
 ## Prerequisites
 
@@ -162,6 +162,48 @@ To deploy updates, manually trigger deployment:
 4. Check "Deploy telemetry server to remote server"
 5. Click "Run workflow"
 
+## Docker Deployment
+
+You can build and run the telemetry server using Docker locally or on a remote host.
+
+### Build
+
+- Using Docker Compose (recommended):
+
+```bash
+docker compose -f telemetry/docker-compose.yml build telemetry-server
+```
+
+- Using docker build directly (from the repository root):
+
+```bash
+docker build -t seaweedfs-telemetry \
+  -f telemetry/server/Dockerfile \
+  .
+```
+
+### Run
+
+- With Docker Compose:
+
+```bash
+docker compose -f telemetry/docker-compose.yml up -d telemetry-server
+```
+
+- With docker run:
+
+```bash
+docker run -d --name telemetry-server \
+  -p 8080:8080 \
+  seaweedfs-telemetry
+```
+
+Notes:
+
+- The container runs as a non-root user by default.
+- The image listens on port `8080` inside the container. Map it with `-p <host_port>:8080`.
+- You can pass flags to the server by appending them after the image name, e.g. `docker run -d -p 8353:8080 seaweedfs-telemetry -port=8353 -dashboard=false`.
+
 ## Server Directory Structure
 
 After setup, the remote server will have:
@@ -199,12 +241,19 @@ sudo systemctl start telemetry.service
 
 ## Accessing the Service
 
-After deployment, the telemetry server will be available at:
+After deployment, the telemetry server will be available at (default ports shown; adjust if you override with `-port`):
 
-- **Dashboard**: `http://your-server:8353`
-- **API**: `http://your-server:8353/api/*`
-- **Metrics**: `http://your-server:8353/metrics`
-- **Health Check**: `http://your-server:8353/health`
+- Docker default: `8080`
+  - **Dashboard**: `http://your-server:8080`
+  - **API**: `http://your-server:8080/api/*`
+  - **Metrics**: `http://your-server:8080/metrics`
+  - **Health Check**: `http://your-server:8080/health`
+
+- Systemd example (if you configured a different port, e.g. `8353`):
+  - **Dashboard**: `http://your-server:8353`
+  - **API**: `http://your-server:8353/api/*`
+  - **Metrics**: `http://your-server:8353/metrics`
+  - **Health Check**: `http://your-server:8353/health`
 
 ## Optional: Prometheus and Grafana Integration
 

telemetry/README.md

@@ -75,11 +75,11 @@ message TelemetryData {
 ```bash
 # Clone and start the complete monitoring stack
 git clone https://github.com/seaweedfs/seaweedfs.git
-cd seaweedfs/telemetry
-docker-compose up -d
+cd seaweedfs
+docker compose -f telemetry/docker-compose.yml up -d
 
 # Or run the server directly
-cd server
+cd telemetry/server
 go run . -port=8080 -dashboard=true
 ```
 
@@ -183,7 +183,9 @@ GET /metrics
 version: '3.8'
 services:
   telemetry-server:
-    build: ./server
+    build:
+      context: ../
+      dockerfile: telemetry/server/Dockerfile
     ports:
       - "8080:8080"
     command: ["-port=8080", "-dashboard=true", "-cleanup=24h"]
@@ -208,18 +210,17 @@ services:
 
 ```bash
 # Deploy the stack
-docker-compose up -d
+docker compose -f telemetry/docker-compose.yml up -d
 
 # Scale telemetry server if needed
-docker-compose up -d --scale telemetry-server=3
+docker compose -f telemetry/docker-compose.yml up -d --scale telemetry-server=3
 ```
 
 ### Server Only
 
 ```bash
-# Build and run telemetry server
-cd server
-docker build -t seaweedfs-telemetry .
+# Build and run telemetry server (build from repo root to include all sources)
+docker build -t seaweedfs-telemetry -f telemetry/server/Dockerfile .
 docker run -p 8080:8080 seaweedfs-telemetry -port=8080 -dashboard=true
 ```
 

telemetry/docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3.8'
-
 services:
   telemetry-server:
-    build: ./server
+    build:
+      context: ../
+      dockerfile: telemetry/server/Dockerfile
     ports:
       - "8080:8080"
     command: [

telemetry/server/Dockerfile

@@ -1,18 +1,26 @@
-FROM golang:1.21-alpine AS builder
+FROM golang:1.25-alpine AS builder
 
 WORKDIR /app
+
 COPY go.mod go.sum ./
 RUN go mod download
 
+WORKDIR /app
 COPY . .
+
+WORKDIR /app/telemetry/server
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o telemetry-server .
 
 FROM alpine:latest
-RUN apk --no-cache add ca-certificates
-WORKDIR /root/
+RUN apk --no-cache add ca-certificates \
+    && addgroup -S appgroup \
+    && adduser -S appuser -G appgroup
 
-COPY --from=builder /app/telemetry-server .
+WORKDIR /home/appuser/
+COPY --from=builder /app/telemetry/server/telemetry-server .
 
 EXPOSE 8080
 
-CMD ["./telemetry-server"] 
+USER appuser
+
+CMD ["./telemetry-server"]

telemetry/server/go.mod

@@ -0,0 +1,24 @@
+module github.com/seaweedfs/seaweedfs/telemetry/server
+
+go 1.25
+
+toolchain go1.25.0
+
+require (
+	github.com/prometheus/client_golang v1.23.2
+	github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
+	google.golang.org/protobuf v1.36.8
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.17.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/sys v0.36.0 // indirect
+)
+
+replace github.com/seaweedfs/seaweedfs => ../..

telemetry/server/go.sum

@@ -1,31 +1,45 @@
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
+github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

telemetry/test/integration.go

@@ -24,58 +24,58 @@ const (
 )
 
 func main() {
-	fmt.Println("🧪 Starting SeaweedFS Telemetry Integration Test")
+	fmt.Println("Starting SeaweedFS Telemetry Integration Test")
 
 	// Start telemetry server
-	fmt.Println("📡 Starting telemetry server...")
+	fmt.Println("Starting telemetry server...")
 	serverCmd, err := startTelemetryServer()
 	if err != nil {
-		log.Fatalf("❌ Failed to start telemetry server: %v", err)
+		log.Fatalf("Failed to start telemetry server: %v", err)
 	}
 	defer stopServer(serverCmd)
 
 	// Wait for server to start
 	if !waitForServer(serverURL+"/health", 15*time.Second) {
-		log.Fatal("❌ Telemetry server failed to start")
+		log.Fatal("Telemetry server failed to start")
 	}
-	fmt.Println("✅ Telemetry server started successfully")
+	fmt.Println("Telemetry server started successfully")
 
 	// Test protobuf marshaling first
-	fmt.Println("🔧 Testing protobuf marshaling...")
+	fmt.Println("Testing protobuf marshaling...")
 	if err := testProtobufMarshaling(); err != nil {
-		log.Fatalf("❌ Protobuf marshaling test failed: %v", err)
+		log.Fatalf("Protobuf marshaling test failed: %v", err)
 	}
-	fmt.Println("✅ Protobuf marshaling test passed")
+	fmt.Println("Protobuf marshaling test passed")
 
 	// Test protobuf client
-	fmt.Println("🔄 Testing protobuf telemetry client...")
+	fmt.Println("Testing protobuf telemetry client...")
 	if err := testTelemetryClient(); err != nil {
-		log.Fatalf("❌ Telemetry client test failed: %v", err)
+		log.Fatalf("Telemetry client test failed: %v", err)
 	}
-	fmt.Println("✅ Telemetry client test passed")
+	fmt.Println("Telemetry client test passed")
 
 	// Test server metrics endpoint
-	fmt.Println("📊 Testing Prometheus metrics endpoint...")
+	fmt.Println("Testing Prometheus metrics endpoint...")
 	if err := testMetricsEndpoint(); err != nil {
-		log.Fatalf("❌ Metrics endpoint test failed: %v", err)
+		log.Fatalf("Metrics endpoint test failed: %v", err)
 	}
-	fmt.Println("✅ Metrics endpoint test passed")
+	fmt.Println("Metrics endpoint test passed")
 
 	// Test stats API
-	fmt.Println("📈 Testing stats API...")
+	fmt.Println("Testing stats API...")
 	if err := testStatsAPI(); err != nil {
-		log.Fatalf("❌ Stats API test failed: %v", err)
+		log.Fatalf("Stats API test failed: %v", err)
 	}
-	fmt.Println("✅ Stats API test passed")
+	fmt.Println("Stats API test passed")
 
 	// Test instances API
-	fmt.Println("📋 Testing instances API...")
+	fmt.Println("Testing instances API...")
 	if err := testInstancesAPI(); err != nil {
-		log.Fatalf("❌ Instances API test failed: %v", err)
+		log.Fatalf("Instances API test failed: %v", err)
 	}
-	fmt.Println("✅ Instances API test passed")
+	fmt.Println("Instances API test passed")
 
-	fmt.Println("🎉 All telemetry integration tests passed!")
+	fmt.Println("All telemetry integration tests passed!")
 }
 
 func startTelemetryServer() (*exec.Cmd, error) {
@@ -126,7 +126,7 @@ func waitForServer(url string, timeout time.Duration) bool {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	fmt.Printf("⏳ Waiting for server at %s...\n", url)
+	fmt.Printf("Waiting for server at %s...\n", url)
 
 	for {
 		select {

test/erasure_coding/ec_integration_test.go

@@ -141,9 +141,9 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
 
 		// The key test: check if the fix prevents the timing issue
 		if contains(outputStr, "Collecting volume locations") && contains(outputStr, "before EC encoding") {
-			t.Logf("✅ FIX DETECTED: Volume locations collected BEFORE EC encoding (timing bug prevented)")
+			t.Logf("FIX DETECTED: Volume locations collected BEFORE EC encoding (timing bug prevented)")
 		} else {
-			t.Logf("❌ NO FIX: Volume locations NOT collected before EC encoding (timing bug may occur)")
+			t.Logf("NO FIX: Volume locations NOT collected before EC encoding (timing bug may occur)")
 		}
 
 		// After EC encoding, try to get volume locations - this simulates the timing bug
@@ -324,10 +324,10 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
 
 		// Check if our fix is present (volume locations collected before EC encoding)
 		if contains(outputStr, "Collecting volume locations") && contains(outputStr, "before EC encoding") {
-			t.Logf("✅ TIMING FIX DETECTED: Volume locations collected BEFORE EC encoding")
+			t.Logf("TIMING FIX DETECTED: Volume locations collected BEFORE EC encoding")
 			t.Logf("This prevents the race condition where master metadata is updated before location collection")
 		} else {
-			t.Logf("❌ NO TIMING FIX: Volume locations may be collected AFTER master metadata update")
+			t.Logf("NO TIMING FIX: Volume locations may be collected AFTER master metadata update")
 			t.Logf("This could cause the race condition leading to cleanup failure and storage waste")
 		}
 
@@ -473,7 +473,7 @@ func findWeedBinary() string {
 func waitForServer(address string, timeout time.Duration) error {
 	start := time.Now()
 	for time.Since(start) < timeout {
-		if conn, err := grpc.Dial(address, grpc.WithInsecure()); err == nil {
+		if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil {
 			conn.Close()
 			return nil
 		}

test/fuse_integration/Makefile

@@ -2,7 +2,7 @@
 
 # Configuration
 WEED_BINARY := weed
-GO_VERSION := 1.21
+GO_VERSION := 1.24
 TEST_TIMEOUT := 30m
 COVERAGE_FILE := coverage.out
 

test/fuse_integration/README.md

@@ -232,7 +232,7 @@ jobs:
 
 ### Docker Testing
 ```dockerfile
-FROM golang:1.21
+FROM golang:1.24
 RUN apt-get update && apt-get install -y fuse
 COPY . /seaweedfs
 WORKDIR /seaweedfs

test/fuse_integration/working_demo_test.go

@@ -118,8 +118,8 @@ func (f *DemoFuseTestFramework) Cleanup() {
 // using local filesystem instead of actual FUSE mounts. It exists to prove
 // the framework concept works while Go module conflicts are resolved.
 func TestFrameworkDemo(t *testing.T) {
-	t.Log("🚀 SeaweedFS FUSE Integration Testing Framework Demo")
-	t.Log("ℹ️  This demo simulates FUSE operations using local filesystem")
+	t.Log("SeaweedFS FUSE Integration Testing Framework Demo")
+	t.Log("This demo simulates FUSE operations using local filesystem")
 
 	// Initialize demo framework
 	framework := NewDemoFuseTestFramework(t, DefaultDemoTestConfig())
@@ -133,7 +133,7 @@ func TestFrameworkDemo(t *testing.T) {
 		if config.Replication != "000" {
 			t.Errorf("Expected replication '000', got %s", config.Replication)
 		}
-		t.Log("✅ Configuration validation passed")
+		t.Log("Configuration validation passed")
 	})
 
 	t.Run("BasicFileOperations", func(t *testing.T) {
@@ -141,16 +141,16 @@ func TestFrameworkDemo(t *testing.T) {
 		content := []byte("Hello, SeaweedFS FUSE Testing!")
 		filename := "demo_test.txt"
 
-		t.Log("📝 Creating test file...")
+		t.Log("Creating test file...")
 		framework.CreateTestFile(filename, content)
 
-		t.Log("🔍 Verifying file exists...")
+		t.Log("Verifying file exists...")
 		framework.AssertFileExists(filename)
 
-		t.Log("📖 Verifying file content...")
+		t.Log("Verifying file content...")
 		framework.AssertFileContent(filename, content)
 
-		t.Log("✅ Basic file operations test passed")
+		t.Log("Basic file operations test passed")
 	})
 
 	t.Run("LargeFileSimulation", func(t *testing.T) {
@@ -162,21 +162,21 @@ func TestFrameworkDemo(t *testing.T) {
 
 		filename := "large_file_demo.dat"
 
-		t.Log("📝 Creating large test file (1MB)...")
+		t.Log("Creating large test file (1MB)...")
 		framework.CreateTestFile(filename, largeContent)
 
-		t.Log("🔍 Verifying large file...")
+		t.Log("Verifying large file...")
 		framework.AssertFileExists(filename)
 		framework.AssertFileContent(filename, largeContent)
 
-		t.Log("✅ Large file operations test passed")
+		t.Log("Large file operations test passed")
 	})
 
 	t.Run("ConcurrencySimulation", func(t *testing.T) {
 		// Simulate concurrent operations
 		numFiles := 5
 
-		t.Logf("📝 Creating %d files concurrently...", numFiles)
+		t.Logf("Creating %d files concurrently...", numFiles)
 
 		for i := 0; i < numFiles; i++ {
 			filename := filepath.Join("concurrent", "file_"+string(rune('A'+i))+".txt")
@@ -186,11 +186,11 @@ func TestFrameworkDemo(t *testing.T) {
 			framework.AssertFileExists(filename)
 		}
 
-		t.Log("✅ Concurrent operations simulation passed")
+		t.Log("Concurrent operations simulation passed")
 	})
 
-	t.Log("🎉 Framework demonstration completed successfully!")
-	t.Log("📊 This DEMO shows the planned FUSE testing capabilities:")
+	t.Log("Framework demonstration completed successfully!")
+	t.Log("This DEMO shows the planned FUSE testing capabilities:")
 	t.Log("   • Automated cluster setup/teardown (simulated)")
 	t.Log("   • File operations testing (local filesystem simulation)")
 	t.Log("   • Directory operations testing (planned)")
@@ -198,5 +198,5 @@ func TestFrameworkDemo(t *testing.T) {
 	t.Log("   • Concurrent operations testing (simulated)")
 	t.Log("   • Error scenario validation (planned)")
 	t.Log("   • Performance validation (planned)")
-	t.Log("ℹ️  Full framework available in framework.go (pending module resolution)")
+	t.Log("Full framework available in framework.go (pending module resolution)")
 }

test/kafka/Dockerfile.kafka-gateway

@@ -0,0 +1,56 @@
+# Dockerfile for Kafka Gateway Integration Testing
+FROM golang:1.24-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git make gcc musl-dev sqlite-dev
+
+# Set working directory
+WORKDIR /app
+
+# Copy go mod files
+COPY go.mod go.sum ./
+
+# Download dependencies
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Build the weed binary with Kafka gateway support
+RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o weed ./weed
+
+# Final stage
+FROM alpine:latest
+
+# Install runtime dependencies
+RUN apk --no-cache add ca-certificates wget curl netcat-openbsd sqlite
+
+# Create non-root user
+RUN addgroup -g 1000 seaweedfs && \
+    adduser -D -s /bin/sh -u 1000 -G seaweedfs seaweedfs
+
+# Set working directory
+WORKDIR /usr/bin
+
+# Copy binary from builder
+COPY --from=builder /app/weed .
+
+# Create data directory
+RUN mkdir -p /data && chown seaweedfs:seaweedfs /data
+
+# Copy startup script
+COPY test/kafka/scripts/kafka-gateway-start.sh /usr/bin/kafka-gateway-start.sh
+RUN chmod +x /usr/bin/kafka-gateway-start.sh
+
+# Switch to non-root user
+USER seaweedfs
+
+# Expose Kafka protocol port and pprof port
+EXPOSE 9093 10093
+
+# Health check
+HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=3 \
+  CMD nc -z localhost 9093 || exit 1
+
+# Default command
+CMD ["/usr/bin/kafka-gateway-start.sh"]

test/kafka/Dockerfile.seaweedfs

@@ -0,0 +1,25 @@
+# Dockerfile for building SeaweedFS components from the current workspace
+FROM golang:1.24-alpine AS builder
+
+RUN apk add --no-cache git make gcc musl-dev sqlite-dev
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=1 GOOS=linux go build -o /out/weed ./weed
+
+FROM alpine:latest
+
+RUN apk --no-cache add ca-certificates curl wget netcat-openbsd sqlite
+
+COPY --from=builder /out/weed /usr/bin/weed
+
+WORKDIR /data
+
+EXPOSE 9333 19333 8080 18080 8888 18888 16777 17777
+
+ENTRYPOINT ["/usr/bin/weed"]

test/kafka/Dockerfile.test-setup

@@ -0,0 +1,29 @@
+# Dockerfile for Kafka Integration Test Setup
+FROM golang:1.24-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git make gcc musl-dev
+
+# Copy repository
+WORKDIR /app
+COPY . .
+
+# Build test setup utility from the test module
+WORKDIR /app/test/kafka
+RUN go mod download
+RUN CGO_ENABLED=1 GOOS=linux go build -o /out/test-setup ./cmd/setup
+
+# Final stage
+FROM alpine:latest
+
+# Install runtime dependencies
+RUN apk --no-cache add ca-certificates curl jq netcat-openbsd
+
+# Copy binary from builder
+COPY --from=builder /out/test-setup /usr/bin/test-setup
+
+# Make executable
+RUN chmod +x /usr/bin/test-setup
+
+# Default command
+CMD ["/usr/bin/test-setup"]

test/kafka/Makefile

@@ -0,0 +1,206 @@
+# Kafka Integration Testing Makefile - Refactored
+# This replaces the existing Makefile with better organization
+
+# Configuration
+ifndef DOCKER_COMPOSE
+DOCKER_COMPOSE := $(if $(shell command -v docker-compose 2>/dev/null),docker-compose,docker compose)
+endif
+TEST_TIMEOUT ?= 10m
+KAFKA_BOOTSTRAP_SERVERS ?= localhost:9092
+KAFKA_GATEWAY_URL ?= localhost:9093
+SCHEMA_REGISTRY_URL ?= http://localhost:8081
+
+# Colors for output
+BLUE := \033[36m
+GREEN := \033[32m
+YELLOW := \033[33m
+RED := \033[31m
+NC := \033[0m # No Color
+
+.PHONY: help setup test clean logs status
+
+help: ## Show this help message
+	@echo "$(BLUE)SeaweedFS Kafka Integration Testing - Refactored$(NC)"
+	@echo ""
+	@echo "Available targets:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  $(GREEN)%-20s$(NC) %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+# Environment Setup
+setup: ## Set up test environment (Kafka + Schema Registry + SeaweedFS)
+	@echo "$(YELLOW)Setting up Kafka integration test environment...$(NC)"
+	@$(DOCKER_COMPOSE) up -d
+	@echo "$(BLUE)Waiting for all services to be ready...$(NC)"
+	@./scripts/wait-for-services.sh
+	@echo "$(GREEN)Test environment ready!$(NC)"
+
+setup-schemas: setup ## Set up test environment and register schemas
+	@echo "$(YELLOW)Registering test schemas...$(NC)"
+	@$(DOCKER_COMPOSE) --profile setup run --rm test-setup
+	@echo "$(GREEN)Schemas registered!$(NC)"
+
+# Test Categories
+test: test-unit test-integration test-e2e ## Run all tests
+
+test-unit: ## Run unit tests
+	@echo "$(YELLOW)Running unit tests...$(NC)"
+	@go test -v -timeout=$(TEST_TIMEOUT) ./unit/...
+
+test-integration: ## Run integration tests
+	@echo "$(YELLOW)Running integration tests...$(NC)"
+	@go test -v -timeout=$(TEST_TIMEOUT) ./integration/...
+
+test-e2e: setup-schemas ## Run end-to-end tests
+	@echo "$(YELLOW)Running end-to-end tests...$(NC)"
+	@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
+		KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
+		SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
+		go test -v -timeout=$(TEST_TIMEOUT) ./e2e/...
+
+test-docker: setup-schemas ## Run Docker integration tests
+	@echo "$(YELLOW)Running Docker integration tests...$(NC)"
+	@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
+		KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
+		SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
+		go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run Docker
+
+# Schema-specific tests
+test-schema: setup-schemas ## Run schema registry integration tests
+	@echo "$(YELLOW)Running schema registry integration tests...$(NC)"
+	@SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
+		go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run Schema
+
+# Client-specific tests
+test-sarama: setup-schemas ## Run Sarama client tests
+	@echo "$(YELLOW)Running Sarama client tests...$(NC)"
+	@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
+		KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
+		go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run Sarama
+
+test-kafka-go: setup-schemas ## Run kafka-go client tests
+	@echo "$(YELLOW)Running kafka-go client tests...$(NC)"
+	@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
+		KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
+		go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run KafkaGo
+
+# Performance tests
+test-performance: setup-schemas ## Run performance benchmarks
+	@echo "$(YELLOW)Running Kafka performance benchmarks...$(NC)"
+	@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
+		KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
+		SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
+		go test -v -timeout=$(TEST_TIMEOUT) -bench=. ./...
+
+# Development targets
+dev-kafka: ## Start only Kafka ecosystem for development
+	@$(DOCKER_COMPOSE) up -d zookeeper kafka schema-registry
+	@sleep 20
+	@$(DOCKER_COMPOSE) --profile setup run --rm test-setup
+
+dev-seaweedfs: ## Start only SeaweedFS for development
+	@$(DOCKER_COMPOSE) up -d seaweedfs-master seaweedfs-volume seaweedfs-filer seaweedfs-mq-broker seaweedfs-mq-agent
+
+dev-gateway: dev-seaweedfs ## Start Kafka Gateway for development
+	@$(DOCKER_COMPOSE) up -d kafka-gateway
+
+dev-test: dev-kafka ## Quick test with just Kafka ecosystem
+	@SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) go test -v -timeout=30s ./unit/...
+
+# Cleanup
+clean: ## Clean up test environment
+	@echo "$(YELLOW)Cleaning up test environment...$(NC)"
+	@$(DOCKER_COMPOSE) down -v --remove-orphans
+	@docker system prune -f
+	@echo "$(GREEN)Environment cleaned up!$(NC)"
+
+# Monitoring and debugging
+logs: ## Show logs from all services
+	@$(DOCKER_COMPOSE) logs --tail=50 -f
+
+logs-kafka: ## Show Kafka logs
+	@$(DOCKER_COMPOSE) logs --tail=100 -f kafka
+
+logs-schema-registry: ## Show Schema Registry logs
+	@$(DOCKER_COMPOSE) logs --tail=100 -f schema-registry
+
+logs-seaweedfs: ## Show SeaweedFS logs
+	@$(DOCKER_COMPOSE) logs --tail=100 -f seaweedfs-master seaweedfs-volume seaweedfs-filer seaweedfs-mq-broker seaweedfs-mq-agent
+
+logs-gateway: ## Show Kafka Gateway logs
+	@$(DOCKER_COMPOSE) logs --tail=100 -f kafka-gateway
+
+status: ## Show status of all services
+	@echo "$(BLUE)Service Status:$(NC)"
+	@$(DOCKER_COMPOSE) ps
+	@echo ""
+	@echo "$(BLUE)Kafka Status:$(NC)"
+	@curl -s http://localhost:9092 > /dev/null && echo "Kafka accessible" || echo "Kafka not accessible"
+	@echo ""
+	@echo "$(BLUE)Schema Registry Status:$(NC)"
+	@curl -s $(SCHEMA_REGISTRY_URL)/subjects > /dev/null && echo "Schema Registry accessible" || echo "Schema Registry not accessible"
+	@echo ""
+	@echo "$(BLUE)Kafka Gateway Status:$(NC)"
+	@nc -z localhost 9093 && echo "Kafka Gateway accessible" || echo "Kafka Gateway not accessible"
+
+debug: ## Debug test environment
+	@echo "$(BLUE)Debug Information:$(NC)"
+	@echo "Kafka Bootstrap Servers: $(KAFKA_BOOTSTRAP_SERVERS)"
+	@echo "Schema Registry URL: $(SCHEMA_REGISTRY_URL)"
+	@echo "Kafka Gateway URL: $(KAFKA_GATEWAY_URL)"
+	@echo ""
+	@echo "Docker Compose Status:"
+	@$(DOCKER_COMPOSE) ps
+	@echo ""
+	@echo "Network connectivity:"
+	@docker network ls | grep kafka-integration-test || echo "No Kafka test network found"
+	@echo ""
+	@echo "Schema Registry subjects:"
+	@curl -s $(SCHEMA_REGISTRY_URL)/subjects 2>/dev/null || echo "Schema Registry not accessible"
+
+# Utility targets
+install-deps: ## Install required dependencies
+	@echo "$(YELLOW)Installing test dependencies...$(NC)"
+	@which docker > /dev/null || (echo "$(RED)Docker not found$(NC)" && exit 1)
+	@which docker-compose > /dev/null || (echo "$(RED)Docker Compose not found$(NC)" && exit 1)
+	@which curl > /dev/null || (echo "$(RED)curl not found$(NC)" && exit 1)
+	@which nc > /dev/null || (echo "$(RED)netcat not found$(NC)" && exit 1)
+	@echo "$(GREEN)All dependencies available$(NC)"
+
+check-env: ## Check test environment setup
+	@echo "$(BLUE)Environment Check:$(NC)"
+	@echo "KAFKA_BOOTSTRAP_SERVERS: $(KAFKA_BOOTSTRAP_SERVERS)"
+	@echo "SCHEMA_REGISTRY_URL: $(SCHEMA_REGISTRY_URL)"
+	@echo "KAFKA_GATEWAY_URL: $(KAFKA_GATEWAY_URL)"
+	@echo "TEST_TIMEOUT: $(TEST_TIMEOUT)"
+	@make install-deps
+
+# CI targets
+ci-test: ## Run tests in CI environment
+	@echo "$(YELLOW)Running CI tests...$(NC)"
+	@make setup-schemas
+	@make test-unit
+	@make test-integration
+	@make clean
+
+ci-e2e: ## Run end-to-end tests in CI
+	@echo "$(YELLOW)Running CI end-to-end tests...$(NC)"
+	@make test-e2e
+	@make clean
+
+# Interactive targets
+shell-kafka: ## Open shell in Kafka container
+	@$(DOCKER_COMPOSE) exec kafka bash
+
+shell-gateway: ## Open shell in Kafka Gateway container
+	@$(DOCKER_COMPOSE) exec kafka-gateway sh
+
+topics: ## List Kafka topics
+	@$(DOCKER_COMPOSE) exec kafka kafka-topics --list --bootstrap-server localhost:29092
+
+create-topic: ## Create a test topic (usage: make create-topic TOPIC=my-topic)
+	@$(DOCKER_COMPOSE) exec kafka kafka-topics --create --topic $(TOPIC) --bootstrap-server localhost:29092 --partitions 3 --replication-factor 1
+
+produce: ## Produce test messages (usage: make produce TOPIC=my-topic)
+	@$(DOCKER_COMPOSE) exec kafka kafka-console-producer --bootstrap-server localhost:29092 --topic $(TOPIC)
+
+consume: ## Consume messages (usage: make consume TOPIC=my-topic)
+	@$(DOCKER_COMPOSE) exec kafka kafka-console-consumer --bootstrap-server localhost:29092 --topic $(TOPIC) --from-beginning

test/kafka/README.md

@@ -0,0 +1,156 @@
+# Kafka Gateway Tests with SMQ Integration
+
+This directory contains tests for the SeaweedFS Kafka Gateway with full SeaweedMQ (SMQ) integration.
+
+## Test Types
+
+### **Unit Tests** (`./unit/`)
+- Basic gateway functionality
+- Protocol compatibility 
+- No SeaweedFS backend required
+- Uses mock handlers
+
+### **Integration Tests** (`./integration/`)
+- **Mock Mode** (default): Uses in-memory handlers for protocol testing
+- **SMQ Mode** (with `SEAWEEDFS_MASTERS`): Uses real SeaweedFS backend for full integration
+
+### **E2E Tests** (`./e2e/`)
+- End-to-end workflows
+- Automatically detects SMQ availability
+- Falls back to mock mode if SMQ unavailable
+
+## Running Tests Locally
+
+### Quick Protocol Testing (Mock Mode)
+```bash
+# Run all integration tests with mock backend
+cd test/kafka
+go test ./integration/...
+
+# Run specific test
+go test -v ./integration/ -run TestClientCompatibility
+```
+
+### Full Integration Testing (SMQ Mode)
+Requires running SeaweedFS instance:
+
+1. **Start SeaweedFS with MQ support:**
+```bash
+# Terminal 1: Start SeaweedFS server
+weed server -ip="127.0.0.1" -ip.bind="0.0.0.0" -dir=/tmp/seaweedfs-data -master.port=9333 -volume.port=8081 -filer.port=8888 -filer=true
+
+# Terminal 2: Start MQ broker  
+weed mq.broker -master="127.0.0.1:9333" -ip="127.0.0.1" -port=17777
+```
+
+2. **Run tests with SMQ backend:**
+```bash
+cd test/kafka
+SEAWEEDFS_MASTERS=127.0.0.1:9333 go test ./integration/...
+
+# Run specific SMQ integration tests
+SEAWEEDFS_MASTERS=127.0.0.1:9333 go test -v ./integration/ -run TestSMQIntegration
+```
+
+### Test Broker Startup
+If you're having broker startup issues:
+```bash
+# Debug broker startup locally
+./scripts/test-broker-startup.sh
+```
+
+## CI/CD Integration
+
+### GitHub Actions Jobs
+
+1. **Unit Tests** - Fast protocol tests with mock backend
+2. **Integration Tests** - Mock mode by default  
+3. **E2E Tests (with SMQ)** - Full SeaweedFS + MQ broker stack
+4. **Client Compatibility (with SMQ)** - Tests different Kafka clients against real backend
+5. **Consumer Group Tests (with SMQ)** - Tests consumer group persistence
+6. **SMQ Integration Tests** - Dedicated SMQ-specific functionality tests
+
+### What Gets Tested with SMQ
+
+When `SEAWEEDFS_MASTERS` is available, tests exercise:
+
+- **Real Message Persistence** - Messages stored in SeaweedFS volumes  
+- **Offset Persistence** - Consumer group offsets stored in SeaweedFS filer  
+- **Topic Persistence** - Topic metadata persisted in SeaweedFS filer  
+- **Consumer Group Coordination** - Distributed coordinator assignment  
+- **Cross-Client Compatibility** - Sarama, kafka-go with real backend  
+- **Broker Discovery** - Gateway discovers MQ brokers via masters  
+
+## Test Infrastructure
+
+### `testutil.NewGatewayTestServerWithSMQ(t, mode)`
+
+Smart gateway creation that automatically:
+- Detects SMQ availability via `SEAWEEDFS_MASTERS`
+- Uses production handler when available
+- Falls back to mock when unavailable  
+- Provides timeout protection against hanging
+
+**Modes:**
+- `SMQRequired` - Skip test if SMQ unavailable
+- `SMQAvailable` - Use SMQ if available, otherwise mock
+- `SMQUnavailable` - Always use mock
+
+### Timeout Protection
+
+Gateway creation includes timeout protection to prevent CI hanging:
+- 20 second timeout for `SMQRequired` mode
+- 15 second timeout for `SMQAvailable` mode  
+- Clear error messages when broker discovery fails
+
+## Debugging Failed Tests
+
+### CI Logs to Check
+1. **"SeaweedFS master is up"** - Master started successfully
+2. **"SeaweedFS filer is up"** - Filer ready  
+3. **"SeaweedFS MQ broker is up"** - Broker started successfully
+4. **Broker/Server logs** - Shown on broker startup failure
+
+### Local Debugging
+1. Run `./scripts/test-broker-startup.sh` to test broker startup
+2. Check logs at `/tmp/weed-*.log` 
+3. Test individual components:
+   ```bash
+   # Test master
+   curl http://127.0.0.1:9333/cluster/status
+   
+   # Test filer  
+   curl http://127.0.0.1:8888/status
+   
+   # Test broker
+   nc -z 127.0.0.1 17777
+   ```
+
+### Common Issues
+- **Broker fails to start**: Check filer is ready before starting broker
+- **Gateway timeout**: Broker discovery fails, check broker is accessible  
+- **Test hangs**: Timeout protection not working, reduce timeout values
+
+## Architecture
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Kafka Client  │───▶│  Kafka Gateway  │───▶│ SeaweedMQ Broker│
+│   (Sarama,      │    │   (Protocol     │    │   (Message      │
+│    kafka-go)    │    │    Handler)     │    │   Persistence)  │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                                │                      │
+                                ▼                      ▼
+                       ┌─────────────────┐    ┌─────────────────┐
+                       │ SeaweedFS Filer │    │ SeaweedFS Master│
+                       │ (Offset Storage)│    │ (Coordination)  │
+                       └─────────────────┘    └─────────────────┘
+                                │                      │
+                                ▼                      ▼  
+                       ┌─────────────────────────────────────────┐
+                       │        SeaweedFS Volumes                │
+                       │      (Message Storage)                  │
+                       └─────────────────────────────────────────┘
+```
+
+This architecture ensures full integration testing of the entire Kafka → SeaweedFS message path.

test/kafka/cmd/setup/main.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"time"
+)
+
+// Schema represents a schema registry schema
+type Schema struct {
+	Subject string `json:"subject"`
+	Version int    `json:"version"`
+	Schema  string `json:"schema"`
+}
+
+// SchemaResponse represents the response from schema registry
+type SchemaResponse struct {
+	ID int `json:"id"`
+}
+
+func main() {
+	log.Println("Setting up Kafka integration test environment...")
+
+	kafkaBootstrap := getEnv("KAFKA_BOOTSTRAP_SERVERS", "kafka:29092")
+	schemaRegistryURL := getEnv("SCHEMA_REGISTRY_URL", "http://schema-registry:8081")
+	kafkaGatewayURL := getEnv("KAFKA_GATEWAY_URL", "kafka-gateway:9093")
+
+	log.Printf("Kafka Bootstrap Servers: %s", kafkaBootstrap)
+	log.Printf("Schema Registry URL: %s", schemaRegistryURL)
+	log.Printf("Kafka Gateway URL: %s", kafkaGatewayURL)
+
+	// Wait for services to be ready
+	waitForHTTPService("Schema Registry", schemaRegistryURL+"/subjects")
+	waitForTCPService("Kafka Gateway", kafkaGatewayURL) // TCP connectivity check for Kafka protocol
+
+	// Register test schemas
+	if err := registerSchemas(schemaRegistryURL); err != nil {
+		log.Fatalf("Failed to register schemas: %v", err)
+	}
+
+	log.Println("Test environment setup completed successfully!")
+}
+
+func getEnv(key, defaultValue string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return defaultValue
+}
+
+func waitForHTTPService(name, url string) {
+	log.Printf("Waiting for %s to be ready...", name)
+	for i := 0; i < 60; i++ { // Wait up to 60 seconds
+		resp, err := http.Get(url)
+		if err == nil && resp.StatusCode < 400 {
+			resp.Body.Close()
+			log.Printf("%s is ready", name)
+			return
+		}
+		if resp != nil {
+			resp.Body.Close()
+		}
+		time.Sleep(1 * time.Second)
+	}
+	log.Fatalf("%s is not ready after 60 seconds", name)
+}
+
+func waitForTCPService(name, address string) {
+	log.Printf("Waiting for %s to be ready...", name)
+	for i := 0; i < 60; i++ { // Wait up to 60 seconds
+		conn, err := net.DialTimeout("tcp", address, 2*time.Second)
+		if err == nil {
+			conn.Close()
+			log.Printf("%s is ready", name)
+			return
+		}
+		time.Sleep(1 * time.Second)
+	}
+	log.Fatalf("%s is not ready after 60 seconds", name)
+}
+
+func registerSchemas(registryURL string) error {
+	schemas := []Schema{
+		{
+			Subject: "user-value",
+			Schema: `{
+				"type": "record",
+				"name": "User",
+				"fields": [
+					{"name": "id", "type": "int"},
+					{"name": "name", "type": "string"},
+					{"name": "email", "type": ["null", "string"], "default": null}
+				]
+			}`,
+		},
+		{
+			Subject: "user-event-value",
+			Schema: `{
+				"type": "record",
+				"name": "UserEvent",
+				"fields": [
+					{"name": "userId", "type": "int"},
+					{"name": "eventType", "type": "string"},
+					{"name": "timestamp", "type": "long"},
+					{"name": "data", "type": ["null", "string"], "default": null}
+				]
+			}`,
+		},
+		{
+			Subject: "log-entry-value",
+			Schema: `{
+				"type": "record",
+				"name": "LogEntry",
+				"fields": [
+					{"name": "level", "type": "string"},
+					{"name": "message", "type": "string"},
+					{"name": "timestamp", "type": "long"},
+					{"name": "service", "type": "string"},
+					{"name": "metadata", "type": {"type": "map", "values": "string"}}
+				]
+			}`,
+		},
+	}
+
+	for _, schema := range schemas {
+		if err := registerSchema(registryURL, schema); err != nil {
+			return fmt.Errorf("failed to register schema %s: %w", schema.Subject, err)
+		}
+		log.Printf("Registered schema: %s", schema.Subject)
+	}
+
+	return nil
+}
+
+func registerSchema(registryURL string, schema Schema) error {
+	url := fmt.Sprintf("%s/subjects/%s/versions", registryURL, schema.Subject)
+
+	payload := map[string]interface{}{
+		"schema": schema.Schema,
+	}
+
+	jsonData, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Post(url, "application/vnd.schemaregistry.v1+json", bytes.NewBuffer(jsonData))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
+	}
+
+	var response SchemaResponse
+	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		return err
+	}
+
+	log.Printf("Schema %s registered with ID: %d", schema.Subject, response.ID)
+	return nil
+}

test/kafka/docker-compose.yml

@@ -0,0 +1,325 @@
+x-seaweedfs-build: &seaweedfs-build
+  build:
+    context: ../..
+    dockerfile: test/kafka/Dockerfile.seaweedfs
+  image: kafka-seaweedfs-dev
+
+services:
+  # Zookeeper for Kafka
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.4.0
+    container_name: kafka-zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "2181"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    networks:
+      - kafka-test-net
+
+  # Kafka Broker
+  kafka:
+    image: confluentinc/cp-kafka:7.4.0
+    container_name: kafka-broker
+    ports:
+      - "9092:9092"
+      - "29092:29092"
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_NUM_PARTITIONS: 3
+      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
+    healthcheck:
+      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:29092"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    networks:
+      - kafka-test-net
+
+  # Schema Registry
+  schema-registry:
+    image: confluentinc/cp-schema-registry:7.4.0
+    container_name: kafka-schema-registry
+    ports:
+      - "8081:8081"
+    depends_on:
+      kafka:
+        condition: service_healthy
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092
+      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
+      SCHEMA_REGISTRY_DEBUG: "true"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+    networks:
+      - kafka-test-net
+
+  # SeaweedFS Master
+  seaweedfs-master:
+    <<: *seaweedfs-build
+    container_name: seaweedfs-master
+    ports:
+      - "9333:9333"
+      - "19333:19333"  # gRPC port
+    command: 
+      - master
+      - -ip=seaweedfs-master
+      - -port=9333
+      - -port.grpc=19333
+      - -volumeSizeLimitMB=1024
+      - -defaultReplication=000
+    volumes:
+      - seaweedfs-master-data:/data
+    healthcheck:
+      test: ["CMD-SHELL", "wget --quiet --tries=1 --spider http://seaweedfs-master:9333/cluster/status || curl -sf http://seaweedfs-master:9333/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 20s
+    networks:
+      - kafka-test-net
+
+  # SeaweedFS Volume Server
+  seaweedfs-volume:
+    <<: *seaweedfs-build
+    container_name: seaweedfs-volume
+    ports:
+      - "8080:8080"
+      - "18080:18080"  # gRPC port
+    command:
+      - volume
+      - -mserver=seaweedfs-master:9333
+      - -ip=seaweedfs-volume
+      - -port=8080
+      - -port.grpc=18080
+      - -publicUrl=seaweedfs-volume:8080
+      - -preStopSeconds=1
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+    volumes:
+      - seaweedfs-volume-data:/data
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://seaweedfs-volume:8080/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    networks:
+      - kafka-test-net
+
+  # SeaweedFS Filer
+  seaweedfs-filer:
+    <<: *seaweedfs-build
+    container_name: seaweedfs-filer
+    ports:
+      - "8888:8888"
+      - "18888:18888"  # gRPC port
+    command:
+      - filer
+      - -master=seaweedfs-master:9333
+      - -ip=seaweedfs-filer
+      - -port=8888
+      - -port.grpc=18888
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+      seaweedfs-volume:
+        condition: service_healthy
+    volumes:
+      - seaweedfs-filer-data:/data
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://seaweedfs-filer:8888/"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+    networks:
+      - kafka-test-net
+
+  # SeaweedFS MQ Broker
+  seaweedfs-mq-broker:
+    <<: *seaweedfs-build
+    container_name: seaweedfs-mq-broker
+    ports:
+      - "17777:17777"  # MQ Broker port
+      - "18777:18777"  # pprof profiling port
+    command:
+      - mq.broker
+      - -master=seaweedfs-master:9333
+      - -ip=seaweedfs-mq-broker
+      - -port=17777
+      - -port.pprof=18777
+    depends_on:
+      seaweedfs-filer:
+        condition: service_healthy
+    volumes:
+      - seaweedfs-mq-data:/data
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "17777"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 20s
+    networks:
+      - kafka-test-net
+
+  # SeaweedFS MQ Agent
+  seaweedfs-mq-agent:
+    <<: *seaweedfs-build
+    container_name: seaweedfs-mq-agent
+    ports:
+      - "16777:16777"  # MQ Agent port
+    command:
+      - mq.agent
+      - -broker=seaweedfs-mq-broker:17777
+      - -ip=0.0.0.0
+      - -port=16777
+    depends_on:
+      seaweedfs-mq-broker:
+        condition: service_healthy
+    volumes:
+      - seaweedfs-mq-data:/data
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "16777"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 25s
+    networks:
+      - kafka-test-net
+
+  # Kafka Gateway (SeaweedFS with Kafka protocol)
+  kafka-gateway:
+    build:
+      context: ../..  # Build from project root
+      dockerfile: test/kafka/Dockerfile.kafka-gateway
+    container_name: kafka-gateway
+    ports:
+      - "9093:9093"  # Kafka protocol port
+      - "10093:10093"  # pprof profiling port
+    depends_on:
+      seaweedfs-mq-agent:
+        condition: service_healthy
+      schema-registry:
+        condition: service_healthy
+    environment:
+      - SEAWEEDFS_MASTERS=seaweedfs-master:9333
+      - SEAWEEDFS_FILER_GROUP=
+      - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+      - KAFKA_PORT=9093
+      - PPROF_PORT=10093
+    volumes:
+      - kafka-gateway-data:/data
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "9093"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    networks:
+      - kafka-test-net
+
+  # Test Data Setup Service
+  test-setup:
+    build:
+      context: ../..
+      dockerfile: test/kafka/Dockerfile.test-setup
+    container_name: kafka-test-setup
+    depends_on:
+      kafka:
+        condition: service_healthy
+      schema-registry:
+        condition: service_healthy
+      kafka-gateway:
+        condition: service_healthy
+    environment:
+      - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
+      - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+      - KAFKA_GATEWAY_URL=kafka-gateway:9093
+    networks:
+      - kafka-test-net
+    restart: "no"  # Run once to set up test data
+    profiles:
+      - setup  # Only start when explicitly requested
+
+  # Kafka Producer for Testing
+  kafka-producer:
+    image: confluentinc/cp-kafka:7.4.0
+    container_name: kafka-producer
+    depends_on:
+      kafka:
+        condition: service_healthy
+      schema-registry:
+        condition: service_healthy
+    environment:
+      - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
+      - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+    networks:
+      - kafka-test-net
+    profiles:
+      - producer  # Only start when explicitly requested
+    command: >
+      sh -c "
+        echo 'Creating test topics...';
+        kafka-topics --create --topic test-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
+        kafka-topics --create --topic avro-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
+        kafka-topics --create --topic schema-test --bootstrap-server kafka:29092 --partitions 1 --replication-factor 1 --if-not-exists;
+        echo 'Topics created successfully';
+        kafka-topics --list --bootstrap-server kafka:29092;
+      "
+
+  # Kafka Consumer for Testing
+  kafka-consumer:
+    image: confluentinc/cp-kafka:7.4.0
+    container_name: kafka-consumer
+    depends_on:
+      kafka:
+        condition: service_healthy
+    environment:
+      - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
+    networks:
+      - kafka-test-net
+    profiles:
+      - consumer  # Only start when explicitly requested
+    command: >
+      kafka-console-consumer
+      --bootstrap-server kafka:29092
+      --topic test-topic
+      --from-beginning
+      --max-messages 10
+
+volumes:
+  seaweedfs-master-data:
+  seaweedfs-volume-data:
+  seaweedfs-filer-data:
+  seaweedfs-mq-data:
+  kafka-gateway-data:
+
+networks:
+  kafka-test-net:
+    driver: bridge
+    name: kafka-integration-test

test/kafka/e2e/comprehensive_test.go

@@ -0,0 +1,131 @@
+package e2e
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+// TestComprehensiveE2E tests complete end-to-end workflows
+// This test will use SMQ backend if SEAWEEDFS_MASTERS is available, otherwise mock
+func TestComprehensiveE2E(t *testing.T) {
+	gateway := testutil.NewGatewayTestServerWithSMQ(t, testutil.SMQAvailable)
+	defer gateway.CleanupAndClose()
+
+	addr := gateway.StartAndWait()
+
+	// Log which backend we're using
+	if gateway.IsSMQMode() {
+		t.Logf("Running comprehensive E2E tests with SMQ backend")
+	} else {
+		t.Logf("Running comprehensive E2E tests with mock backend")
+	}
+
+	// Create topics for different test scenarios
+	topics := []string{
+		testutil.GenerateUniqueTopicName("e2e-kafka-go"),
+		testutil.GenerateUniqueTopicName("e2e-sarama"),
+		testutil.GenerateUniqueTopicName("e2e-mixed"),
+	}
+	gateway.AddTestTopics(topics...)
+
+	t.Run("KafkaGo_to_KafkaGo", func(t *testing.T) {
+		testKafkaGoToKafkaGo(t, addr, topics[0])
+	})
+
+	t.Run("Sarama_to_Sarama", func(t *testing.T) {
+		testSaramaToSarama(t, addr, topics[1])
+	})
+
+	t.Run("KafkaGo_to_Sarama", func(t *testing.T) {
+		testKafkaGoToSarama(t, addr, topics[2])
+	})
+
+	t.Run("Sarama_to_KafkaGo", func(t *testing.T) {
+		testSaramaToKafkaGo(t, addr, topics[2])
+	})
+}
+
+func testKafkaGoToKafkaGo(t *testing.T, addr, topic string) {
+	client := testutil.NewKafkaGoClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Generate test messages
+	messages := msgGen.GenerateKafkaGoMessages(2)
+
+	// Produce with kafka-go
+	err := client.ProduceMessages(topic, messages)
+	testutil.AssertNoError(t, err, "kafka-go produce failed")
+
+	// Consume with kafka-go
+	consumed, err := client.ConsumeMessages(topic, len(messages))
+	testutil.AssertNoError(t, err, "kafka-go consume failed")
+
+	// Validate message content
+	err = testutil.ValidateKafkaGoMessageContent(messages, consumed)
+	testutil.AssertNoError(t, err, "Message content validation failed")
+
+	t.Logf("kafka-go to kafka-go test PASSED")
+}
+
+func testSaramaToSarama(t *testing.T, addr, topic string) {
+	client := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Generate test messages
+	messages := msgGen.GenerateStringMessages(2)
+
+	// Produce with Sarama
+	err := client.ProduceMessages(topic, messages)
+	testutil.AssertNoError(t, err, "Sarama produce failed")
+
+	// Consume with Sarama
+	consumed, err := client.ConsumeMessages(topic, 0, len(messages))
+	testutil.AssertNoError(t, err, "Sarama consume failed")
+
+	// Validate message content
+	err = testutil.ValidateMessageContent(messages, consumed)
+	testutil.AssertNoError(t, err, "Message content validation failed")
+
+	t.Logf("Sarama to Sarama test PASSED")
+}
+
+func testKafkaGoToSarama(t *testing.T, addr, topic string) {
+	kafkaGoClient := testutil.NewKafkaGoClient(t, addr)
+	saramaClient := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Produce with kafka-go
+	messages := msgGen.GenerateKafkaGoMessages(2)
+	err := kafkaGoClient.ProduceMessages(topic, messages)
+	testutil.AssertNoError(t, err, "kafka-go produce failed")
+
+	// Consume with Sarama
+	consumed, err := saramaClient.ConsumeMessages(topic, 0, len(messages))
+	testutil.AssertNoError(t, err, "Sarama consume failed")
+
+	// Validate that we got the expected number of messages
+	testutil.AssertEqual(t, len(messages), len(consumed), "Message count mismatch")
+
+	t.Logf("kafka-go to Sarama test PASSED")
+}
+
+func testSaramaToKafkaGo(t *testing.T, addr, topic string) {
+	kafkaGoClient := testutil.NewKafkaGoClient(t, addr)
+	saramaClient := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Produce with Sarama
+	messages := msgGen.GenerateStringMessages(2)
+	err := saramaClient.ProduceMessages(topic, messages)
+	testutil.AssertNoError(t, err, "Sarama produce failed")
+
+	// Consume with kafka-go
+	consumed, err := kafkaGoClient.ConsumeMessages(topic, len(messages))
+	testutil.AssertNoError(t, err, "kafka-go consume failed")
+
+	// Validate that we got the expected number of messages
+	testutil.AssertEqual(t, len(messages), len(consumed), "Message count mismatch")
+
+	t.Logf("Sarama to kafka-go test PASSED")
+}

test/kafka/e2e/offset_management_test.go

@@ -0,0 +1,130 @@
+package e2e
+
+import (
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+// TestOffsetManagement tests end-to-end offset management scenarios
+// This test will use SMQ backend if SEAWEEDFS_MASTERS is available, otherwise mock
+func TestOffsetManagement(t *testing.T) {
+	gateway := testutil.NewGatewayTestServerWithSMQ(t, testutil.SMQAvailable)
+	defer gateway.CleanupAndClose()
+
+	addr := gateway.StartAndWait()
+
+	// If schema registry is configured, ensure gateway is in schema mode and log
+	if v := os.Getenv("SCHEMA_REGISTRY_URL"); v != "" {
+		t.Logf("Schema Registry detected at %s - running offset tests in schematized mode", v)
+	}
+
+	// Log which backend we're using
+	if gateway.IsSMQMode() {
+		t.Logf("Running offset management tests with SMQ backend - offsets will be persisted")
+	} else {
+		t.Logf("Running offset management tests with mock backend - offsets are in-memory only")
+	}
+
+	topic := testutil.GenerateUniqueTopicName("offset-management")
+	groupID := testutil.GenerateUniqueGroupID("offset-test-group")
+
+	gateway.AddTestTopic(topic)
+
+	t.Run("BasicOffsetCommitFetch", func(t *testing.T) {
+		testBasicOffsetCommitFetch(t, addr, topic, groupID)
+	})
+
+	t.Run("ConsumerGroupResumption", func(t *testing.T) {
+		testConsumerGroupResumption(t, addr, topic, groupID+"2")
+	})
+}
+
+func testBasicOffsetCommitFetch(t *testing.T, addr, topic, groupID string) {
+	client := testutil.NewKafkaGoClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Produce test messages
+	if url := os.Getenv("SCHEMA_REGISTRY_URL"); url != "" {
+		if id, err := testutil.EnsureValueSchema(t, url, topic); err == nil {
+			t.Logf("Ensured value schema id=%d for subject %s-value", id, topic)
+		} else {
+			t.Logf("Schema registration failed (non-fatal for test): %v", err)
+		}
+	}
+	messages := msgGen.GenerateKafkaGoMessages(5)
+	err := client.ProduceMessages(topic, messages)
+	testutil.AssertNoError(t, err, "Failed to produce offset test messages")
+
+	// Phase 1: Consume first 3 messages and commit offsets
+	t.Logf("=== Phase 1: Consuming first 3 messages ===")
+	consumed1, err := client.ConsumeWithGroup(topic, groupID, 3)
+	testutil.AssertNoError(t, err, "Failed to consume first batch")
+	testutil.AssertEqual(t, 3, len(consumed1), "Should consume exactly 3 messages")
+
+	// Phase 2: Create new consumer with same group ID - should resume from committed offset
+	t.Logf("=== Phase 2: Resuming from committed offset ===")
+	consumed2, err := client.ConsumeWithGroup(topic, groupID, 2)
+	testutil.AssertNoError(t, err, "Failed to consume remaining messages")
+	testutil.AssertEqual(t, 2, len(consumed2), "Should consume remaining 2 messages")
+
+	// Verify that we got all messages without duplicates
+	totalConsumed := len(consumed1) + len(consumed2)
+	testutil.AssertEqual(t, len(messages), totalConsumed, "Should consume all messages exactly once")
+
+	t.Logf("SUCCESS: Offset management test completed - consumed %d + %d messages", len(consumed1), len(consumed2))
+}
+
+func testConsumerGroupResumption(t *testing.T, addr, topic, groupID string) {
+	client := testutil.NewKafkaGoClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Produce messages
+	t.Logf("=== Phase 1: Producing 4 messages to topic %s ===", topic)
+	messages := msgGen.GenerateKafkaGoMessages(4)
+	err := client.ProduceMessages(topic, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages for resumption test")
+	t.Logf("Successfully produced %d messages", len(messages))
+
+	// Consume some messages
+	t.Logf("=== Phase 2: First consumer - consuming 2 messages with group %s ===", groupID)
+	consumed1, err := client.ConsumeWithGroup(topic, groupID, 2)
+	testutil.AssertNoError(t, err, "Failed to consume first batch")
+	t.Logf("First consumer consumed %d messages:", len(consumed1))
+	for i, msg := range consumed1 {
+		t.Logf("  Message %d: offset=%d, partition=%d, value=%s", i, msg.Offset, msg.Partition, string(msg.Value))
+	}
+
+	// Simulate consumer restart by consuming remaining messages with same group ID
+	t.Logf("=== Phase 3: Second consumer (simulated restart) - consuming remaining messages with same group %s ===", groupID)
+	consumed2, err := client.ConsumeWithGroup(topic, groupID, 2)
+	testutil.AssertNoError(t, err, "Failed to consume after restart")
+	t.Logf("Second consumer consumed %d messages:", len(consumed2))
+	for i, msg := range consumed2 {
+		t.Logf("  Message %d: offset=%d, partition=%d, value=%s", i, msg.Offset, msg.Partition, string(msg.Value))
+	}
+
+	// Verify total consumption
+	totalConsumed := len(consumed1) + len(consumed2)
+	t.Logf("=== Verification: Total consumed %d messages (expected %d) ===", totalConsumed, len(messages))
+
+	// Check for duplicates
+	offsetsSeen := make(map[int64]bool)
+	duplicateCount := 0
+	for _, msg := range append(consumed1, consumed2...) {
+		if offsetsSeen[msg.Offset] {
+			t.Logf("WARNING: Duplicate offset detected: %d", msg.Offset)
+			duplicateCount++
+		}
+		offsetsSeen[msg.Offset] = true
+	}
+
+	if duplicateCount > 0 {
+		t.Logf("ERROR: Found %d duplicate messages", duplicateCount)
+	}
+
+	testutil.AssertEqual(t, len(messages), totalConsumed, "Should consume all messages after restart")
+
+	t.Logf("SUCCESS: Consumer group resumption test completed - no duplicates, all messages consumed exactly once")
+}

test/kafka/go.mod

@@ -0,0 +1,258 @@
+module github.com/seaweedfs/seaweedfs/test/kafka
+
+go 1.24.0
+
+toolchain go1.24.7
+
+require (
+	github.com/IBM/sarama v1.46.0
+	github.com/linkedin/goavro/v2 v2.14.0
+	github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
+	github.com/segmentio/kafka-go v0.4.49
+	github.com/stretchr/testify v1.11.1
+	google.golang.org/grpc v1.75.1
+)
+
+replace github.com/seaweedfs/seaweedfs => ../../
+
+require (
+	cloud.google.com/go/auth v0.16.5 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
+	github.com/Files-com/files-sdk-go/v3 v3.2.218 // indirect
+	github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
+	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
+	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
+	github.com/ProtonMail/go-crypto v1.3.0 // indirect
+	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
+	github.com/ProtonMail/go-srp v0.0.7 // indirect
+	github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
+	github.com/PuerkitoBio/goquery v1.10.3 // indirect
+	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+	github.com/aws/aws-sdk-go v1.55.8 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.39.2 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect
+	github.com/aws/smithy-go v1.23.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bradenaw/juniper v0.15.3 // indirect
+	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
+	github.com/buengese/sgzip v0.1.1 // indirect
+	github.com/bufbuild/protocompile v0.14.1 // indirect
+	github.com/calebcase/tmpfile v1.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/cloudinary/cloudinary-go/v2 v2.12.0 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
+	github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
+	github.com/cognusion/imaging v1.0.2 // indirect
+	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/creasty/defaults v1.8.0 // indirect
+	github.com/cronokirby/saferith v0.33.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
+	github.com/eapache/go-resiliency v1.7.0 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/ebitengine/purego v0.9.0 // indirect
+	github.com/emersion/go-message v0.18.2 // indirect
+	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/flynn/noise v1.1.0 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
+	github.com/geoffgarside/ber v1.2.0 // indirect
+	github.com/go-chi/chi/v5 v5.2.2 // indirect
+	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-openapi/errors v0.22.2 // indirect
+	github.com/go-openapi/strfmt v0.23.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.27.0 // indirect
+	github.com/go-resty/resty/v2 v2.16.5 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/gofrs/flock v0.12.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/snappy v1.0.0 // indirect
+	github.com/google/btree v1.1.3 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/gorilla/schema v1.4.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
+	github.com/henrybear327/go-proton-api v1.0.0 // indirect
+	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
+	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+	github.com/jcmturner/gofork v1.7.6 // indirect
+	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
+	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
+	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/jhump/protoreflect v1.17.0 // indirect
+	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
+	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
+	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
+	github.com/klauspost/compress v1.18.1 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/klauspost/reedsolomon v1.12.5 // indirect
+	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
+	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lanrat/extsort v1.4.0 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/lpar/date v1.0.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/ncw/swift/v2 v2.0.4 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.98.0 // indirect
+	github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
+	github.com/panjf2000/ants/v2 v2.11.3 // indirect
+	github.com/parquet-go/parquet-go v0.25.1 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
+	github.com/peterh/liner v1.2.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/sftp v1.13.9 // indirect
+	github.com/pkg/xattr v0.4.12 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.17.0 // indirect
+	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
+	github.com/rclone/rclone v1.71.1 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
+	github.com/rdleal/intervalst v1.5.0 // indirect
+	github.com/relvacode/iso8601 v1.6.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rfjakob/eme v1.1.2 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
+	github.com/sagikazarmark/locafero v0.11.0 // indirect
+	github.com/samber/lo v1.51.0 // indirect
+	github.com/seaweedfs/goexif v1.0.3 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.9 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
+	github.com/smarty/assertions v1.16.0 // indirect
+	github.com/sony/gobreaker v1.0.0 // indirect
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/spf13/viper v1.21.0 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect
+	github.com/tklauser/go-sysconf v0.3.15 // indirect
+	github.com/tklauser/numcpus v0.10.0 // indirect
+	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 // indirect
+	github.com/unknwon/goconfig v1.0.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/viant/ptrie v1.0.1 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
+	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	github.com/zeebo/blake3 v0.2.4 // indirect
+	github.com/zeebo/errs v1.4.0 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.etcd.io/bbolt v1.4.2 // indirect
+	go.mongodb.org/mongo-driver v1.17.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.43.0 // indirect
+	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
+	golang.org/x/image v0.32.0 // indirect
+	golang.org/x/net v0.45.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/term v0.36.0 // indirect
+	golang.org/x/text v0.30.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
+	google.golang.org/api v0.247.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
+	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
+	google.golang.org/protobuf v1.36.9 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/validator.v2 v2.0.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	moul.io/http2curl/v2 v2.3.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+	storj.io/common v0.0.0-20250808122759-804533d519c1 // indirect
+	storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
+	storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
+	storj.io/infectious v0.0.2 // indirect
+	storj.io/picobuf v0.0.4 // indirect
+	storj.io/uplink v1.13.1 // indirect
+)

test/kafka/integration/client_compatibility_test.go

@@ -0,0 +1,549 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/IBM/sarama"
+	"github.com/segmentio/kafka-go"
+
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+// TestClientCompatibility tests compatibility with different Kafka client libraries and versions
+// This test will use SMQ backend if SEAWEEDFS_MASTERS is available, otherwise mock
+func TestClientCompatibility(t *testing.T) {
+	gateway := testutil.NewGatewayTestServerWithSMQ(t, testutil.SMQAvailable)
+	defer gateway.CleanupAndClose()
+
+	addr := gateway.StartAndWait()
+	time.Sleep(200 * time.Millisecond) // Allow gateway to be ready
+
+	// Log which backend we're using
+	if gateway.IsSMQMode() {
+		t.Logf("Running client compatibility tests with SMQ backend")
+	} else {
+		t.Logf("Running client compatibility tests with mock backend")
+	}
+
+	t.Run("SaramaVersionCompatibility", func(t *testing.T) {
+		testSaramaVersionCompatibility(t, addr)
+	})
+
+	t.Run("KafkaGoVersionCompatibility", func(t *testing.T) {
+		testKafkaGoVersionCompatibility(t, addr)
+	})
+
+	t.Run("APIVersionNegotiation", func(t *testing.T) {
+		testAPIVersionNegotiation(t, addr)
+	})
+
+	t.Run("ProducerConsumerCompatibility", func(t *testing.T) {
+		testProducerConsumerCompatibility(t, addr)
+	})
+
+	t.Run("ConsumerGroupCompatibility", func(t *testing.T) {
+		testConsumerGroupCompatibility(t, addr)
+	})
+
+	t.Run("AdminClientCompatibility", func(t *testing.T) {
+		testAdminClientCompatibility(t, addr)
+	})
+}
+
+func testSaramaVersionCompatibility(t *testing.T, addr string) {
+	versions := []sarama.KafkaVersion{
+		sarama.V2_6_0_0,
+		sarama.V2_8_0_0,
+		sarama.V3_0_0_0,
+		sarama.V3_4_0_0,
+	}
+
+	for _, version := range versions {
+		t.Run(fmt.Sprintf("Sarama_%s", version.String()), func(t *testing.T) {
+			config := sarama.NewConfig()
+			config.Version = version
+			config.Producer.Return.Successes = true
+			config.Consumer.Return.Errors = true
+
+			client, err := sarama.NewClient([]string{addr}, config)
+			if err != nil {
+				t.Fatalf("Failed to create Sarama client for version %s: %v", version, err)
+			}
+			defer client.Close()
+
+			// Test basic operations
+			topicName := testutil.GenerateUniqueTopicName(fmt.Sprintf("sarama-%s", version.String()))
+
+			// Test topic creation via admin client
+			admin, err := sarama.NewClusterAdminFromClient(client)
+			if err != nil {
+				t.Fatalf("Failed to create admin client: %v", err)
+			}
+			defer admin.Close()
+
+			topicDetail := &sarama.TopicDetail{
+				NumPartitions:     1,
+				ReplicationFactor: 1,
+			}
+
+			err = admin.CreateTopic(topicName, topicDetail, false)
+			if err != nil {
+				t.Logf("Topic creation failed (may already exist): %v", err)
+			}
+
+			// Test produce
+			producer, err := sarama.NewSyncProducerFromClient(client)
+			if err != nil {
+				t.Fatalf("Failed to create producer: %v", err)
+			}
+			defer producer.Close()
+
+			message := &sarama.ProducerMessage{
+				Topic: topicName,
+				Value: sarama.StringEncoder(fmt.Sprintf("test-message-%s", version.String())),
+			}
+
+			partition, offset, err := producer.SendMessage(message)
+			if err != nil {
+				t.Fatalf("Failed to send message: %v", err)
+			}
+
+			t.Logf("Sarama %s: Message sent to partition %d at offset %d", version, partition, offset)
+
+			// Test consume
+			consumer, err := sarama.NewConsumerFromClient(client)
+			if err != nil {
+				t.Fatalf("Failed to create consumer: %v", err)
+			}
+			defer consumer.Close()
+
+			partitionConsumer, err := consumer.ConsumePartition(topicName, 0, sarama.OffsetOldest)
+			if err != nil {
+				t.Fatalf("Failed to create partition consumer: %v", err)
+			}
+			defer partitionConsumer.Close()
+
+			select {
+			case msg := <-partitionConsumer.Messages():
+				if string(msg.Value) != fmt.Sprintf("test-message-%s", version.String()) {
+					t.Errorf("Message content mismatch: expected %s, got %s",
+						fmt.Sprintf("test-message-%s", version.String()), string(msg.Value))
+				}
+				t.Logf("Sarama %s: Successfully consumed message", version)
+			case err := <-partitionConsumer.Errors():
+				t.Fatalf("Consumer error: %v", err)
+			case <-time.After(5 * time.Second):
+				t.Fatal("Timeout waiting for message")
+			}
+		})
+	}
+}
+
+func testKafkaGoVersionCompatibility(t *testing.T, addr string) {
+	// Test different kafka-go configurations
+	configs := []struct {
+		name         string
+		readerConfig kafka.ReaderConfig
+		writerConfig kafka.WriterConfig
+	}{
+		{
+			name: "kafka-go-default",
+			readerConfig: kafka.ReaderConfig{
+				Brokers:   []string{addr},
+				Partition: 0, // Read from specific partition instead of using consumer group
+			},
+			writerConfig: kafka.WriterConfig{
+				Brokers: []string{addr},
+			},
+		},
+		{
+			name: "kafka-go-with-batching",
+			readerConfig: kafka.ReaderConfig{
+				Brokers:   []string{addr},
+				Partition: 0, // Read from specific partition instead of using consumer group
+				MinBytes:  1,
+				MaxBytes:  10e6,
+			},
+			writerConfig: kafka.WriterConfig{
+				Brokers:      []string{addr},
+				BatchSize:    100,
+				BatchTimeout: 10 * time.Millisecond,
+			},
+		},
+	}
+
+	for _, config := range configs {
+		t.Run(config.name, func(t *testing.T) {
+			topicName := testutil.GenerateUniqueTopicName(config.name)
+
+			// Create topic first using Sarama admin client (kafka-go doesn't have admin client)
+			saramaConfig := sarama.NewConfig()
+			saramaClient, err := sarama.NewClient([]string{addr}, saramaConfig)
+			if err != nil {
+				t.Fatalf("Failed to create Sarama client for topic creation: %v", err)
+			}
+			defer saramaClient.Close()
+
+			admin, err := sarama.NewClusterAdminFromClient(saramaClient)
+			if err != nil {
+				t.Fatalf("Failed to create admin client: %v", err)
+			}
+			defer admin.Close()
+
+			topicDetail := &sarama.TopicDetail{
+				NumPartitions:     1,
+				ReplicationFactor: 1,
+			}
+
+			err = admin.CreateTopic(topicName, topicDetail, false)
+			if err != nil {
+				t.Logf("Topic creation failed (may already exist): %v", err)
+			}
+
+			// Wait for topic to be fully created
+			time.Sleep(200 * time.Millisecond)
+
+			// Configure writer first and write message
+			config.writerConfig.Topic = topicName
+			writer := kafka.NewWriter(config.writerConfig)
+
+			// Test produce
+			produceCtx, produceCancel := context.WithTimeout(context.Background(), 15*time.Second)
+			defer produceCancel()
+
+			message := kafka.Message{
+				Value: []byte(fmt.Sprintf("test-message-%s", config.name)),
+			}
+
+			err = writer.WriteMessages(produceCtx, message)
+			if err != nil {
+				writer.Close()
+				t.Fatalf("Failed to write message: %v", err)
+			}
+
+			// Close writer before reading to ensure flush
+			if err := writer.Close(); err != nil {
+				t.Logf("Warning: writer close error: %v", err)
+			}
+
+			t.Logf("%s: Message written successfully", config.name)
+
+			// Wait for message to be available
+			time.Sleep(100 * time.Millisecond)
+
+			// Configure and create reader
+			config.readerConfig.Topic = topicName
+			config.readerConfig.StartOffset = kafka.FirstOffset
+			reader := kafka.NewReader(config.readerConfig)
+
+			// Test consume with dedicated context
+			consumeCtx, consumeCancel := context.WithTimeout(context.Background(), 15*time.Second)
+
+			msg, err := reader.ReadMessage(consumeCtx)
+			consumeCancel()
+
+			if err != nil {
+				reader.Close()
+				t.Fatalf("Failed to read message: %v", err)
+			}
+
+			if string(msg.Value) != fmt.Sprintf("test-message-%s", config.name) {
+				reader.Close()
+				t.Errorf("Message content mismatch: expected %s, got %s",
+					fmt.Sprintf("test-message-%s", config.name), string(msg.Value))
+			}
+
+			t.Logf("%s: Successfully consumed message", config.name)
+
+			// Close reader and wait for cleanup
+			if err := reader.Close(); err != nil {
+				t.Logf("Warning: reader close error: %v", err)
+			}
+
+			// Give time for background goroutines to clean up
+			time.Sleep(100 * time.Millisecond)
+		})
+	}
+}
+
+func testAPIVersionNegotiation(t *testing.T, addr string) {
+	// Test that clients can negotiate API versions properly
+	config := sarama.NewConfig()
+	config.Version = sarama.V2_8_0_0
+
+	client, err := sarama.NewClient([]string{addr}, config)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Test that the client can get API versions
+	coordinator, err := client.Coordinator("test-group")
+	if err != nil {
+		t.Logf("Coordinator lookup failed (expected for test): %v", err)
+	} else {
+		t.Logf("Successfully found coordinator: %s", coordinator.Addr())
+	}
+
+	// Test metadata request (should work with version negotiation)
+	topics, err := client.Topics()
+	if err != nil {
+		t.Fatalf("Failed to get topics: %v", err)
+	}
+
+	t.Logf("API version negotiation successful, found %d topics", len(topics))
+}
+
+func testProducerConsumerCompatibility(t *testing.T, addr string) {
+	// Test cross-client compatibility: produce with one client, consume with another
+	topicName := testutil.GenerateUniqueTopicName("cross-client-test")
+
+	// Create topic first
+	saramaConfig := sarama.NewConfig()
+	saramaConfig.Producer.Return.Successes = true
+
+	saramaClient, err := sarama.NewClient([]string{addr}, saramaConfig)
+	if err != nil {
+		t.Fatalf("Failed to create Sarama client: %v", err)
+	}
+	defer saramaClient.Close()
+
+	admin, err := sarama.NewClusterAdminFromClient(saramaClient)
+	if err != nil {
+		t.Fatalf("Failed to create admin client: %v", err)
+	}
+	defer admin.Close()
+
+	topicDetail := &sarama.TopicDetail{
+		NumPartitions:     1,
+		ReplicationFactor: 1,
+	}
+
+	err = admin.CreateTopic(topicName, topicDetail, false)
+	if err != nil {
+		t.Logf("Topic creation failed (may already exist): %v", err)
+	}
+
+	// Wait for topic to be fully created
+	time.Sleep(200 * time.Millisecond)
+
+	producer, err := sarama.NewSyncProducerFromClient(saramaClient)
+	if err != nil {
+		t.Fatalf("Failed to create producer: %v", err)
+	}
+	defer producer.Close()
+
+	message := &sarama.ProducerMessage{
+		Topic: topicName,
+		Value: sarama.StringEncoder("cross-client-message"),
+	}
+
+	_, _, err = producer.SendMessage(message)
+	if err != nil {
+		t.Fatalf("Failed to send message with Sarama: %v", err)
+	}
+
+	t.Logf("Produced message with Sarama")
+
+	// Wait for message to be available
+	time.Sleep(100 * time.Millisecond)
+
+	// Consume with kafka-go (without consumer group to avoid offset commit issues)
+	reader := kafka.NewReader(kafka.ReaderConfig{
+		Brokers:     []string{addr},
+		Topic:       topicName,
+		Partition:   0,
+		StartOffset: kafka.FirstOffset,
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	msg, err := reader.ReadMessage(ctx)
+	cancel()
+
+	// Close reader immediately after reading
+	if closeErr := reader.Close(); closeErr != nil {
+		t.Logf("Warning: reader close error: %v", closeErr)
+	}
+
+	if err != nil {
+		t.Fatalf("Failed to read message with kafka-go: %v", err)
+	}
+
+	if string(msg.Value) != "cross-client-message" {
+		t.Errorf("Message content mismatch: expected 'cross-client-message', got '%s'", string(msg.Value))
+	}
+
+	t.Logf("Cross-client compatibility test passed")
+}
+
+func testConsumerGroupCompatibility(t *testing.T, addr string) {
+	// Test consumer group functionality with different clients
+	topicName := testutil.GenerateUniqueTopicName("consumer-group-test")
+
+	// Create topic and produce messages
+	config := sarama.NewConfig()
+	config.Producer.Return.Successes = true
+
+	client, err := sarama.NewClient([]string{addr}, config)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Create topic first
+	admin, err := sarama.NewClusterAdminFromClient(client)
+	if err != nil {
+		t.Fatalf("Failed to create admin client: %v", err)
+	}
+	defer admin.Close()
+
+	topicDetail := &sarama.TopicDetail{
+		NumPartitions:     1,
+		ReplicationFactor: 1,
+	}
+
+	err = admin.CreateTopic(topicName, topicDetail, false)
+	if err != nil {
+		t.Logf("Topic creation failed (may already exist): %v", err)
+	}
+
+	// Wait for topic to be fully created
+	time.Sleep(200 * time.Millisecond)
+
+	producer, err := sarama.NewSyncProducerFromClient(client)
+	if err != nil {
+		t.Fatalf("Failed to create producer: %v", err)
+	}
+	defer producer.Close()
+
+	// Produce test messages
+	for i := 0; i < 5; i++ {
+		message := &sarama.ProducerMessage{
+			Topic: topicName,
+			Value: sarama.StringEncoder(fmt.Sprintf("group-message-%d", i)),
+		}
+
+		_, _, err = producer.SendMessage(message)
+		if err != nil {
+			t.Fatalf("Failed to send message %d: %v", i, err)
+		}
+	}
+
+	t.Logf("Produced 5 messages successfully")
+
+	// Wait for messages to be available
+	time.Sleep(200 * time.Millisecond)
+
+	// Test consumer group with Sarama (kafka-go consumer groups have offset commit issues)
+	consumer, err := sarama.NewConsumerFromClient(client)
+	if err != nil {
+		t.Fatalf("Failed to create consumer: %v", err)
+	}
+	defer consumer.Close()
+
+	partitionConsumer, err := consumer.ConsumePartition(topicName, 0, sarama.OffsetOldest)
+	if err != nil {
+		t.Fatalf("Failed to create partition consumer: %v", err)
+	}
+	defer partitionConsumer.Close()
+
+	messagesReceived := 0
+	timeout := time.After(30 * time.Second)
+
+	for messagesReceived < 5 {
+		select {
+		case msg := <-partitionConsumer.Messages():
+			t.Logf("Received message %d: %s", messagesReceived, string(msg.Value))
+			messagesReceived++
+		case err := <-partitionConsumer.Errors():
+			t.Logf("Consumer error (continuing): %v", err)
+		case <-timeout:
+			t.Fatalf("Timeout waiting for messages, received %d out of 5", messagesReceived)
+		}
+	}
+
+	t.Logf("Consumer group compatibility test passed: received %d messages", messagesReceived)
+}
+
+func testAdminClientCompatibility(t *testing.T, addr string) {
+	// Test admin operations with different clients
+	config := sarama.NewConfig()
+	config.Version = sarama.V2_8_0_0
+	config.Admin.Timeout = 30 * time.Second
+
+	client, err := sarama.NewClient([]string{addr}, config)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	admin, err := sarama.NewClusterAdminFromClient(client)
+	if err != nil {
+		t.Fatalf("Failed to create admin client: %v", err)
+	}
+	defer admin.Close()
+
+	// Test topic operations
+	topicName := testutil.GenerateUniqueTopicName("admin-test")
+
+	topicDetail := &sarama.TopicDetail{
+		NumPartitions:     2,
+		ReplicationFactor: 1,
+	}
+
+	err = admin.CreateTopic(topicName, topicDetail, false)
+	if err != nil {
+		t.Logf("Topic creation failed (may already exist): %v", err)
+	}
+
+	// Wait for topic to be fully created and propagated
+	time.Sleep(500 * time.Millisecond)
+
+	// List topics with retry logic
+	var topics map[string]sarama.TopicDetail
+	maxRetries := 3
+	for i := 0; i < maxRetries; i++ {
+		topics, err = admin.ListTopics()
+		if err == nil {
+			break
+		}
+		t.Logf("List topics attempt %d failed: %v, retrying...", i+1, err)
+		time.Sleep(time.Duration(500*(i+1)) * time.Millisecond)
+	}
+
+	if err != nil {
+		t.Fatalf("Failed to list topics after %d attempts: %v", maxRetries, err)
+	}
+
+	found := false
+	for topic := range topics {
+		if topic == topicName {
+			found = true
+			t.Logf("Found created topic: %s", topicName)
+			break
+		}
+	}
+
+	if !found {
+		// Log all topics for debugging
+		allTopics := make([]string, 0, len(topics))
+		for topic := range topics {
+			allTopics = append(allTopics, topic)
+		}
+		t.Logf("Available topics: %v", allTopics)
+		t.Errorf("Created topic %s not found in topic list", topicName)
+	}
+
+	// Test describe consumer groups (if supported)
+	groups, err := admin.ListConsumerGroups()
+	if err != nil {
+		t.Logf("List consumer groups failed (may not be implemented): %v", err)
+	} else {
+		t.Logf("Found %d consumer groups", len(groups))
+	}
+
+	t.Logf("Admin client compatibility test passed")
+}

test/kafka/integration/consumer_groups_test.go

@@ -0,0 +1,351 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/IBM/sarama"
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+// TestConsumerGroups tests consumer group functionality
+// This test requires SeaweedFS masters to be running and will skip if not available
+func TestConsumerGroups(t *testing.T) {
+	gateway := testutil.NewGatewayTestServerWithSMQ(t, testutil.SMQRequired)
+	defer gateway.CleanupAndClose()
+
+	addr := gateway.StartAndWait()
+
+	t.Logf("Running consumer group tests with SMQ backend for offset persistence")
+
+	t.Run("BasicFunctionality", func(t *testing.T) {
+		testConsumerGroupBasicFunctionality(t, addr)
+	})
+
+	t.Run("OffsetCommitAndFetch", func(t *testing.T) {
+		testConsumerGroupOffsetCommitAndFetch(t, addr)
+	})
+
+	t.Run("Rebalancing", func(t *testing.T) {
+		testConsumerGroupRebalancing(t, addr)
+	})
+}
+
+func testConsumerGroupBasicFunctionality(t *testing.T, addr string) {
+	topicName := testutil.GenerateUniqueTopicName("consumer-group-basic")
+	groupID := testutil.GenerateUniqueGroupID("basic-group")
+
+	client := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Create topic and produce messages
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	messages := msgGen.GenerateStringMessages(9) // 3 messages per consumer
+	err = client.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages")
+
+	// Test with multiple consumers in the same group
+	numConsumers := 3
+	handler := &ConsumerGroupHandler{
+		messages: make(chan *sarama.ConsumerMessage, len(messages)),
+		ready:    make(chan bool),
+		t:        t,
+	}
+
+	var wg sync.WaitGroup
+	consumerErrors := make(chan error, numConsumers)
+
+	for i := 0; i < numConsumers; i++ {
+		wg.Add(1)
+		go func(consumerID int) {
+			defer wg.Done()
+
+			consumerGroup, err := sarama.NewConsumerGroup([]string{addr}, groupID, client.GetConfig())
+			if err != nil {
+				consumerErrors <- fmt.Errorf("consumer %d: failed to create consumer group: %v", consumerID, err)
+				return
+			}
+			defer consumerGroup.Close()
+
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+
+			err = consumerGroup.Consume(ctx, []string{topicName}, handler)
+			if err != nil && err != context.DeadlineExceeded {
+				consumerErrors <- fmt.Errorf("consumer %d: consumption error: %v", consumerID, err)
+				return
+			}
+		}(i)
+	}
+
+	// Wait for consumers to be ready
+	readyCount := 0
+	for readyCount < numConsumers {
+		select {
+		case <-handler.ready:
+			readyCount++
+		case <-time.After(5 * time.Second):
+			t.Fatalf("Timeout waiting for consumers to be ready")
+		}
+	}
+
+	// Collect consumed messages
+	consumedMessages := make([]*sarama.ConsumerMessage, 0, len(messages))
+	messageTimeout := time.After(10 * time.Second)
+
+	for len(consumedMessages) < len(messages) {
+		select {
+		case msg := <-handler.messages:
+			consumedMessages = append(consumedMessages, msg)
+		case err := <-consumerErrors:
+			t.Fatalf("Consumer error: %v", err)
+		case <-messageTimeout:
+			t.Fatalf("Timeout waiting for messages. Got %d/%d messages", len(consumedMessages), len(messages))
+		}
+	}
+
+	wg.Wait()
+
+	// Verify all messages were consumed exactly once
+	testutil.AssertEqual(t, len(messages), len(consumedMessages), "Message count mismatch")
+
+	// Verify message uniqueness (no duplicates)
+	messageKeys := make(map[string]bool)
+	for _, msg := range consumedMessages {
+		key := string(msg.Key)
+		if messageKeys[key] {
+			t.Errorf("Duplicate message key: %s", key)
+		}
+		messageKeys[key] = true
+	}
+}
+
+func testConsumerGroupOffsetCommitAndFetch(t *testing.T, addr string) {
+	topicName := testutil.GenerateUniqueTopicName("offset-commit-test")
+	groupID := testutil.GenerateUniqueGroupID("offset-group")
+
+	client := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Create topic and produce messages
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	messages := msgGen.GenerateStringMessages(5)
+	err = client.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages")
+
+	// First consumer: consume first 3 messages and commit offsets
+	handler1 := &OffsetTestHandler{
+		messages:  make(chan *sarama.ConsumerMessage, len(messages)),
+		ready:     make(chan bool),
+		stopAfter: 3,
+		t:         t,
+	}
+
+	consumerGroup1, err := sarama.NewConsumerGroup([]string{addr}, groupID, client.GetConfig())
+	testutil.AssertNoError(t, err, "Failed to create first consumer group")
+
+	ctx1, cancel1 := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel1()
+
+	go func() {
+		err := consumerGroup1.Consume(ctx1, []string{topicName}, handler1)
+		if err != nil && err != context.DeadlineExceeded {
+			t.Logf("First consumer error: %v", err)
+		}
+	}()
+
+	// Wait for first consumer to be ready and consume messages
+	<-handler1.ready
+	consumedCount := 0
+	for consumedCount < 3 {
+		select {
+		case <-handler1.messages:
+			consumedCount++
+		case <-time.After(5 * time.Second):
+			t.Fatalf("Timeout waiting for first consumer messages")
+		}
+	}
+
+	consumerGroup1.Close()
+	cancel1()
+	time.Sleep(500 * time.Millisecond) // Wait for cleanup
+
+	// Stop the first consumer after N messages
+	// Allow a brief moment for commit/heartbeat to flush
+	time.Sleep(1 * time.Second)
+
+	// Start a second consumer in the same group to verify resumption from committed offset
+	handler2 := &OffsetTestHandler{
+		messages:  make(chan *sarama.ConsumerMessage, len(messages)),
+		ready:     make(chan bool),
+		stopAfter: 2,
+		t:         t,
+	}
+	consumerGroup2, err := sarama.NewConsumerGroup([]string{addr}, groupID, client.GetConfig())
+	testutil.AssertNoError(t, err, "Failed to create second consumer group")
+	defer consumerGroup2.Close()
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel2()
+
+	go func() {
+		err := consumerGroup2.Consume(ctx2, []string{topicName}, handler2)
+		if err != nil && err != context.DeadlineExceeded {
+			t.Logf("Second consumer error: %v", err)
+		}
+	}()
+
+	// Wait for second consumer and collect remaining messages
+	<-handler2.ready
+	secondConsumerMessages := make([]*sarama.ConsumerMessage, 0)
+	consumedCount = 0
+	for consumedCount < 2 {
+		select {
+		case msg := <-handler2.messages:
+			consumedCount++
+			secondConsumerMessages = append(secondConsumerMessages, msg)
+		case <-time.After(5 * time.Second):
+			t.Fatalf("Timeout waiting for second consumer messages. Got %d/2", consumedCount)
+		}
+	}
+
+	// Verify second consumer started from correct offset
+	if len(secondConsumerMessages) > 0 {
+		firstMessageOffset := secondConsumerMessages[0].Offset
+		if firstMessageOffset < 3 {
+			t.Fatalf("Second consumer should start from offset >= 3: got %d", firstMessageOffset)
+		}
+	}
+}
+
+func testConsumerGroupRebalancing(t *testing.T, addr string) {
+	topicName := testutil.GenerateUniqueTopicName("rebalancing-test")
+	groupID := testutil.GenerateUniqueGroupID("rebalance-group")
+
+	client := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Create topic with multiple partitions for rebalancing
+	err := client.CreateTopic(topicName, 4, 1) // 4 partitions
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	// Produce messages to all partitions
+	messages := msgGen.GenerateStringMessages(12) // 3 messages per partition
+	for i, msg := range messages {
+		partition := int32(i % 4)
+		err = client.ProduceMessageToPartition(topicName, partition, msg)
+		testutil.AssertNoError(t, err, "Failed to produce message")
+	}
+
+	t.Logf("Produced %d messages across 4 partitions", len(messages))
+
+	// Test scenario 1: Single consumer gets all partitions
+	t.Run("SingleConsumerAllPartitions", func(t *testing.T) {
+		testSingleConsumerAllPartitions(t, addr, topicName, groupID+"-single")
+	})
+
+	// Test scenario 2: Add second consumer, verify rebalancing
+	t.Run("TwoConsumersRebalance", func(t *testing.T) {
+		testTwoConsumersRebalance(t, addr, topicName, groupID+"-two")
+	})
+
+	// Test scenario 3: Remove consumer, verify rebalancing
+	t.Run("ConsumerLeaveRebalance", func(t *testing.T) {
+		testConsumerLeaveRebalance(t, addr, topicName, groupID+"-leave")
+	})
+
+	// Test scenario 4: Multiple consumers join simultaneously
+	t.Run("MultipleConsumersJoin", func(t *testing.T) {
+		testMultipleConsumersJoin(t, addr, topicName, groupID+"-multi")
+	})
+}
+
+// ConsumerGroupHandler implements sarama.ConsumerGroupHandler
+type ConsumerGroupHandler struct {
+	messages  chan *sarama.ConsumerMessage
+	ready     chan bool
+	readyOnce sync.Once
+	t         *testing.T
+}
+
+func (h *ConsumerGroupHandler) Setup(sarama.ConsumerGroupSession) error {
+	h.t.Logf("Consumer group session setup")
+	h.readyOnce.Do(func() {
+		close(h.ready)
+	})
+	return nil
+}
+
+func (h *ConsumerGroupHandler) Cleanup(sarama.ConsumerGroupSession) error {
+	h.t.Logf("Consumer group session cleanup")
+	return nil
+}
+
+func (h *ConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
+	for {
+		select {
+		case message := <-claim.Messages():
+			if message == nil {
+				return nil
+			}
+			h.messages <- message
+			session.MarkMessage(message, "")
+		case <-session.Context().Done():
+			return nil
+		}
+	}
+}
+
+// OffsetTestHandler implements sarama.ConsumerGroupHandler for offset testing
+type OffsetTestHandler struct {
+	messages  chan *sarama.ConsumerMessage
+	ready     chan bool
+	readyOnce sync.Once
+	stopAfter int
+	consumed  int
+	t         *testing.T
+}
+
+func (h *OffsetTestHandler) Setup(sarama.ConsumerGroupSession) error {
+	h.t.Logf("Offset test consumer setup")
+	h.readyOnce.Do(func() {
+		close(h.ready)
+	})
+	return nil
+}
+
+func (h *OffsetTestHandler) Cleanup(sarama.ConsumerGroupSession) error {
+	h.t.Logf("Offset test consumer cleanup")
+	return nil
+}
+
+func (h *OffsetTestHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
+	for {
+		select {
+		case message := <-claim.Messages():
+			if message == nil {
+				return nil
+			}
+			h.consumed++
+			h.messages <- message
+			session.MarkMessage(message, "")
+
+			// Stop after consuming the specified number of messages
+			if h.consumed >= h.stopAfter {
+				h.t.Logf("Stopping consumer after %d messages", h.consumed)
+				// Ensure commits are flushed before exiting the claim
+				session.Commit()
+				return nil
+			}
+		case <-session.Context().Done():
+			return nil
+		}
+	}
+}

test/kafka/integration/docker_test.go

@@ -0,0 +1,216 @@
+package integration
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+// TestDockerIntegration tests the complete Kafka integration using Docker Compose
+func TestDockerIntegration(t *testing.T) {
+	env := testutil.NewDockerEnvironment(t)
+	env.SkipIfNotAvailable(t)
+
+	t.Run("KafkaConnectivity", func(t *testing.T) {
+		env.RequireKafka(t)
+		testDockerKafkaConnectivity(t, env.KafkaBootstrap)
+	})
+
+	t.Run("SchemaRegistryConnectivity", func(t *testing.T) {
+		env.RequireSchemaRegistry(t)
+		testDockerSchemaRegistryConnectivity(t, env.SchemaRegistry)
+	})
+
+	t.Run("KafkaGatewayConnectivity", func(t *testing.T) {
+		env.RequireGateway(t)
+		testDockerKafkaGatewayConnectivity(t, env.KafkaGateway)
+	})
+
+	t.Run("SaramaProduceConsume", func(t *testing.T) {
+		env.RequireKafka(t)
+		testDockerSaramaProduceConsume(t, env.KafkaBootstrap)
+	})
+
+	t.Run("KafkaGoProduceConsume", func(t *testing.T) {
+		env.RequireKafka(t)
+		testDockerKafkaGoProduceConsume(t, env.KafkaBootstrap)
+	})
+
+	t.Run("GatewayProduceConsume", func(t *testing.T) {
+		env.RequireGateway(t)
+		testDockerGatewayProduceConsume(t, env.KafkaGateway)
+	})
+
+	t.Run("CrossClientCompatibility", func(t *testing.T) {
+		env.RequireKafka(t)
+		env.RequireGateway(t)
+		testDockerCrossClientCompatibility(t, env.KafkaBootstrap, env.KafkaGateway)
+	})
+}
+
+func testDockerKafkaConnectivity(t *testing.T, bootstrap string) {
+	client := testutil.NewSaramaClient(t, bootstrap)
+
+	// Test basic connectivity by creating a topic
+	topicName := testutil.GenerateUniqueTopicName("connectivity-test")
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic for connectivity test")
+
+	t.Logf("Kafka connectivity test passed")
+}
+
+func testDockerSchemaRegistryConnectivity(t *testing.T, registryURL string) {
+	// Test basic HTTP connectivity to Schema Registry
+	client := &http.Client{Timeout: 10 * time.Second}
+
+	// Test 1: Check if Schema Registry is responding
+	resp, err := client.Get(registryURL + "/subjects")
+	if err != nil {
+		t.Fatalf("Failed to connect to Schema Registry at %s: %v", registryURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Schema Registry returned status %d, expected 200", resp.StatusCode)
+	}
+
+	// Test 2: Verify response is valid JSON array
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("Failed to read response body: %v", err)
+	}
+
+	var subjects []string
+	if err := json.Unmarshal(body, &subjects); err != nil {
+		t.Fatalf("Schema Registry response is not valid JSON array: %v", err)
+	}
+
+	t.Logf("Schema Registry is accessible with %d subjects", len(subjects))
+
+	// Test 3: Check config endpoint
+	configResp, err := client.Get(registryURL + "/config")
+	if err != nil {
+		t.Fatalf("Failed to get Schema Registry config: %v", err)
+	}
+	defer configResp.Body.Close()
+
+	if configResp.StatusCode != http.StatusOK {
+		t.Fatalf("Schema Registry config endpoint returned status %d", configResp.StatusCode)
+	}
+
+	configBody, err := io.ReadAll(configResp.Body)
+	if err != nil {
+		t.Fatalf("Failed to read config response: %v", err)
+	}
+
+	var config map[string]interface{}
+	if err := json.Unmarshal(configBody, &config); err != nil {
+		t.Fatalf("Schema Registry config response is not valid JSON: %v", err)
+	}
+
+	t.Logf("Schema Registry config: %v", config)
+	t.Logf("Schema Registry connectivity test passed")
+}
+
+func testDockerKafkaGatewayConnectivity(t *testing.T, gatewayURL string) {
+	client := testutil.NewSaramaClient(t, gatewayURL)
+
+	// Test basic connectivity to gateway
+	topicName := testutil.GenerateUniqueTopicName("gateway-connectivity-test")
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic via gateway")
+
+	t.Logf("Kafka Gateway connectivity test passed")
+}
+
+func testDockerSaramaProduceConsume(t *testing.T, bootstrap string) {
+	client := testutil.NewSaramaClient(t, bootstrap)
+	msgGen := testutil.NewMessageGenerator()
+
+	topicName := testutil.GenerateUniqueTopicName("sarama-docker-test")
+
+	// Create topic
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	// Produce and consume messages
+	messages := msgGen.GenerateStringMessages(3)
+	err = client.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages")
+
+	consumed, err := client.ConsumeMessages(topicName, 0, len(messages))
+	testutil.AssertNoError(t, err, "Failed to consume messages")
+
+	err = testutil.ValidateMessageContent(messages, consumed)
+	testutil.AssertNoError(t, err, "Message validation failed")
+
+	t.Logf("Sarama produce/consume test passed")
+}
+
+func testDockerKafkaGoProduceConsume(t *testing.T, bootstrap string) {
+	client := testutil.NewKafkaGoClient(t, bootstrap)
+	msgGen := testutil.NewMessageGenerator()
+
+	topicName := testutil.GenerateUniqueTopicName("kafka-go-docker-test")
+
+	// Create topic
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	// Produce and consume messages
+	messages := msgGen.GenerateKafkaGoMessages(3)
+	err = client.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages")
+
+	consumed, err := client.ConsumeMessages(topicName, len(messages))
+	testutil.AssertNoError(t, err, "Failed to consume messages")
+
+	err = testutil.ValidateKafkaGoMessageContent(messages, consumed)
+	testutil.AssertNoError(t, err, "Message validation failed")
+
+	t.Logf("kafka-go produce/consume test passed")
+}
+
+func testDockerGatewayProduceConsume(t *testing.T, gatewayURL string) {
+	client := testutil.NewSaramaClient(t, gatewayURL)
+	msgGen := testutil.NewMessageGenerator()
+
+	topicName := testutil.GenerateUniqueTopicName("gateway-docker-test")
+
+	// Produce and consume via gateway
+	messages := msgGen.GenerateStringMessages(3)
+	err := client.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages via gateway")
+
+	consumed, err := client.ConsumeMessages(topicName, 0, len(messages))
+	testutil.AssertNoError(t, err, "Failed to consume messages via gateway")
+
+	err = testutil.ValidateMessageContent(messages, consumed)
+	testutil.AssertNoError(t, err, "Message validation failed")
+
+	t.Logf("Gateway produce/consume test passed")
+}
+
+func testDockerCrossClientCompatibility(t *testing.T, kafkaBootstrap, gatewayURL string) {
+	kafkaClient := testutil.NewSaramaClient(t, kafkaBootstrap)
+	msgGen := testutil.NewMessageGenerator()
+
+	topicName := testutil.GenerateUniqueTopicName("cross-client-docker-test")
+
+	// Create topic on Kafka
+	err := kafkaClient.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic on Kafka")
+
+	// Produce to Kafka
+	messages := msgGen.GenerateStringMessages(2)
+	err = kafkaClient.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce to Kafka")
+
+	// This tests the integration between Kafka and the Gateway
+	// In a real scenario, messages would be replicated or bridged
+	t.Logf("Cross-client compatibility test passed")
+}

test/kafka/integration/rebalancing_test.go

@@ -0,0 +1,453 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/IBM/sarama"
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+func testSingleConsumerAllPartitions(t *testing.T, addr, topicName, groupID string) {
+	config := sarama.NewConfig()
+	config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
+	config.Consumer.Offsets.Initial = sarama.OffsetOldest
+	config.Consumer.Return.Errors = true
+
+	client, err := sarama.NewClient([]string{addr}, config)
+	testutil.AssertNoError(t, err, "Failed to create client")
+	defer client.Close()
+
+	consumerGroup, err := sarama.NewConsumerGroupFromClient(groupID, client)
+	testutil.AssertNoError(t, err, "Failed to create consumer group")
+	defer consumerGroup.Close()
+
+	handler := &RebalanceTestHandler{
+		messages:    make(chan *sarama.ConsumerMessage, 20),
+		ready:       make(chan bool),
+		assignments: make(chan []int32, 5),
+		t:           t,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Start consumer
+	go func() {
+		err := consumerGroup.Consume(ctx, []string{topicName}, handler)
+		if err != nil && err != context.DeadlineExceeded {
+			t.Logf("Consumer error: %v", err)
+		}
+	}()
+
+	// Wait for consumer to be ready
+	<-handler.ready
+
+	// Wait for assignment
+	select {
+	case partitions := <-handler.assignments:
+		t.Logf("Single consumer assigned partitions: %v", partitions)
+		if len(partitions) != 4 {
+			t.Errorf("Expected single consumer to get all 4 partitions, got %d", len(partitions))
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for partition assignment")
+	}
+
+	// Consume some messages to verify functionality
+	consumedCount := 0
+	for consumedCount < 4 { // At least one from each partition
+		select {
+		case msg := <-handler.messages:
+			t.Logf("Consumed message from partition %d: %s", msg.Partition, string(msg.Value))
+			consumedCount++
+		case <-time.After(5 * time.Second):
+			t.Logf("Consumed %d messages so far", consumedCount)
+			break
+		}
+	}
+
+	if consumedCount == 0 {
+		t.Error("No messages consumed by single consumer")
+	}
+}
+
+func testTwoConsumersRebalance(t *testing.T, addr, topicName, groupID string) {
+	config := sarama.NewConfig()
+	config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
+	config.Consumer.Offsets.Initial = sarama.OffsetOldest
+	config.Consumer.Return.Errors = true
+
+	// Start first consumer
+	client1, err := sarama.NewClient([]string{addr}, config)
+	testutil.AssertNoError(t, err, "Failed to create client1")
+	defer client1.Close()
+
+	consumerGroup1, err := sarama.NewConsumerGroupFromClient(groupID, client1)
+	testutil.AssertNoError(t, err, "Failed to create consumer group 1")
+	defer consumerGroup1.Close()
+
+	handler1 := &RebalanceTestHandler{
+		messages:    make(chan *sarama.ConsumerMessage, 20),
+		ready:       make(chan bool),
+		assignments: make(chan []int32, 5),
+		t:           t,
+		name:        "Consumer1",
+	}
+
+	ctx1, cancel1 := context.WithTimeout(context.Background(), 45*time.Second)
+	defer cancel1()
+
+	go func() {
+		err := consumerGroup1.Consume(ctx1, []string{topicName}, handler1)
+		if err != nil && err != context.DeadlineExceeded {
+			t.Logf("Consumer1 error: %v", err)
+		}
+	}()
+
+	// Wait for first consumer to be ready and get initial assignment
+	<-handler1.ready
+	select {
+	case partitions := <-handler1.assignments:
+		t.Logf("Consumer1 initial assignment: %v", partitions)
+		if len(partitions) != 4 {
+			t.Errorf("Expected Consumer1 to initially get all 4 partitions, got %d", len(partitions))
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for Consumer1 initial assignment")
+	}
+
+	// Start second consumer
+	client2, err := sarama.NewClient([]string{addr}, config)
+	testutil.AssertNoError(t, err, "Failed to create client2")
+	defer client2.Close()
+
+	consumerGroup2, err := sarama.NewConsumerGroupFromClient(groupID, client2)
+	testutil.AssertNoError(t, err, "Failed to create consumer group 2")
+	defer consumerGroup2.Close()
+
+	handler2 := &RebalanceTestHandler{
+		messages:    make(chan *sarama.ConsumerMessage, 20),
+		ready:       make(chan bool),
+		assignments: make(chan []int32, 5),
+		t:           t,
+		name:        "Consumer2",
+	}
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel2()
+
+	go func() {
+		err := consumerGroup2.Consume(ctx2, []string{topicName}, handler2)
+		if err != nil && err != context.DeadlineExceeded {
+			t.Logf("Consumer2 error: %v", err)
+		}
+	}()
+
+	// Wait for second consumer to be ready
+	<-handler2.ready
+
+	// Wait for rebalancing to occur - both consumers should get new assignments
+	var rebalancedAssignment1, rebalancedAssignment2 []int32
+	
+	// Consumer1 should get a rebalance assignment
+	select {
+	case partitions := <-handler1.assignments:
+		rebalancedAssignment1 = partitions
+		t.Logf("Consumer1 rebalanced assignment: %v", partitions)
+	case <-time.After(15 * time.Second):
+		t.Error("Timeout waiting for Consumer1 rebalance assignment")
+	}
+
+	// Consumer2 should get its assignment
+	select {
+	case partitions := <-handler2.assignments:
+		rebalancedAssignment2 = partitions
+		t.Logf("Consumer2 assignment: %v", partitions)
+	case <-time.After(15 * time.Second):
+		t.Error("Timeout waiting for Consumer2 assignment")
+	}
+
+	// Verify rebalancing occurred correctly
+	totalPartitions := len(rebalancedAssignment1) + len(rebalancedAssignment2)
+	if totalPartitions != 4 {
+		t.Errorf("Expected total of 4 partitions assigned, got %d", totalPartitions)
+	}
+
+	// Each consumer should have at least 1 partition, and no more than 3
+	if len(rebalancedAssignment1) == 0 || len(rebalancedAssignment1) > 3 {
+		t.Errorf("Consumer1 should have 1-3 partitions, got %d", len(rebalancedAssignment1))
+	}
+	if len(rebalancedAssignment2) == 0 || len(rebalancedAssignment2) > 3 {
+		t.Errorf("Consumer2 should have 1-3 partitions, got %d", len(rebalancedAssignment2))
+	}
+
+	// Verify no partition overlap
+	partitionSet := make(map[int32]bool)
+	for _, p := range rebalancedAssignment1 {
+		if partitionSet[p] {
+			t.Errorf("Partition %d assigned to multiple consumers", p)
+		}
+		partitionSet[p] = true
+	}
+	for _, p := range rebalancedAssignment2 {
+		if partitionSet[p] {
+			t.Errorf("Partition %d assigned to multiple consumers", p)
+		}
+		partitionSet[p] = true
+	}
+
+	t.Logf("Rebalancing test completed successfully")
+}
+
+func testConsumerLeaveRebalance(t *testing.T, addr, topicName, groupID string) {
+	config := sarama.NewConfig()
+	config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
+	config.Consumer.Offsets.Initial = sarama.OffsetOldest
+	config.Consumer.Return.Errors = true
+
+	// Start two consumers
+	client1, err := sarama.NewClient([]string{addr}, config)
+	testutil.AssertNoError(t, err, "Failed to create client1")
+	defer client1.Close()
+
+	client2, err := sarama.NewClient([]string{addr}, config)
+	testutil.AssertNoError(t, err, "Failed to create client2")
+	defer client2.Close()
+
+	consumerGroup1, err := sarama.NewConsumerGroupFromClient(groupID, client1)
+	testutil.AssertNoError(t, err, "Failed to create consumer group 1")
+	defer consumerGroup1.Close()
+
+	consumerGroup2, err := sarama.NewConsumerGroupFromClient(groupID, client2)
+	testutil.AssertNoError(t, err, "Failed to create consumer group 2")
+
+	handler1 := &RebalanceTestHandler{
+		messages:    make(chan *sarama.ConsumerMessage, 20),
+		ready:       make(chan bool),
+		assignments: make(chan []int32, 5),
+		t:           t,
+		name:        "Consumer1",
+	}
+
+	handler2 := &RebalanceTestHandler{
+		messages:    make(chan *sarama.ConsumerMessage, 20),
+		ready:       make(chan bool),
+		assignments: make(chan []int32, 5),
+		t:           t,
+		name:        "Consumer2",
+	}
+
+	ctx1, cancel1 := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel1()
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
+
+	// Start both consumers
+	go func() {
+		err := consumerGroup1.Consume(ctx1, []string{topicName}, handler1)
+		if err != nil && err != context.DeadlineExceeded {
+			t.Logf("Consumer1 error: %v", err)
+		}
+	}()
+
+	go func() {
+		err := consumerGroup2.Consume(ctx2, []string{topicName}, handler2)
+		if err != nil && err != context.DeadlineExceeded {
+			t.Logf("Consumer2 error: %v", err)
+		}
+	}()
+
+	// Wait for both consumers to be ready
+	<-handler1.ready
+	<-handler2.ready
+
+	// Wait for initial assignments
+	<-handler1.assignments
+	<-handler2.assignments
+
+	t.Logf("Both consumers started, now stopping Consumer2")
+
+	// Stop second consumer (simulate leave)
+	cancel2()
+	consumerGroup2.Close()
+
+	// Wait for Consumer1 to get rebalanced assignment (should get all partitions)
+	select {
+	case partitions := <-handler1.assignments:
+		t.Logf("Consumer1 rebalanced assignment after Consumer2 left: %v", partitions)
+		if len(partitions) != 4 {
+			t.Errorf("Expected Consumer1 to get all 4 partitions after Consumer2 left, got %d", len(partitions))
+		}
+	case <-time.After(20 * time.Second):
+		t.Error("Timeout waiting for Consumer1 rebalance after Consumer2 left")
+	}
+
+	t.Logf("Consumer leave rebalancing test completed successfully")
+}
+
+func testMultipleConsumersJoin(t *testing.T, addr, topicName, groupID string) {
+	config := sarama.NewConfig()
+	config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
+	config.Consumer.Offsets.Initial = sarama.OffsetOldest
+	config.Consumer.Return.Errors = true
+
+	numConsumers := 4
+	consumers := make([]sarama.ConsumerGroup, numConsumers)
+	clients := make([]sarama.Client, numConsumers)
+	handlers := make([]*RebalanceTestHandler, numConsumers)
+	contexts := make([]context.Context, numConsumers)
+	cancels := make([]context.CancelFunc, numConsumers)
+
+	// Start all consumers simultaneously
+	for i := 0; i < numConsumers; i++ {
+		client, err := sarama.NewClient([]string{addr}, config)
+		testutil.AssertNoError(t, err, fmt.Sprintf("Failed to create client%d", i))
+		clients[i] = client
+
+		consumerGroup, err := sarama.NewConsumerGroupFromClient(groupID, client)
+		testutil.AssertNoError(t, err, fmt.Sprintf("Failed to create consumer group %d", i))
+		consumers[i] = consumerGroup
+
+		handlers[i] = &RebalanceTestHandler{
+			messages:    make(chan *sarama.ConsumerMessage, 20),
+			ready:       make(chan bool),
+			assignments: make(chan []int32, 5),
+			t:           t,
+			name:        fmt.Sprintf("Consumer%d", i),
+		}
+
+		contexts[i], cancels[i] = context.WithTimeout(context.Background(), 45*time.Second)
+
+		go func(idx int) {
+			err := consumers[idx].Consume(contexts[idx], []string{topicName}, handlers[idx])
+			if err != nil && err != context.DeadlineExceeded {
+				t.Logf("Consumer%d error: %v", idx, err)
+			}
+		}(i)
+	}
+
+	// Cleanup
+	defer func() {
+		for i := 0; i < numConsumers; i++ {
+			cancels[i]()
+			consumers[i].Close()
+			clients[i].Close()
+		}
+	}()
+
+	// Wait for all consumers to be ready
+	for i := 0; i < numConsumers; i++ {
+		select {
+		case <-handlers[i].ready:
+			t.Logf("Consumer%d ready", i)
+		case <-time.After(15 * time.Second):
+			t.Fatalf("Timeout waiting for Consumer%d to be ready", i)
+		}
+	}
+
+	// Collect final assignments from all consumers
+	assignments := make([][]int32, numConsumers)
+	for i := 0; i < numConsumers; i++ {
+		select {
+		case partitions := <-handlers[i].assignments:
+			assignments[i] = partitions
+			t.Logf("Consumer%d final assignment: %v", i, partitions)
+		case <-time.After(20 * time.Second):
+			t.Errorf("Timeout waiting for Consumer%d assignment", i)
+		}
+	}
+
+	// Verify all partitions are assigned exactly once
+	assignedPartitions := make(map[int32]int)
+	totalAssigned := 0
+	for i, assignment := range assignments {
+		totalAssigned += len(assignment)
+		for _, partition := range assignment {
+			assignedPartitions[partition]++
+			if assignedPartitions[partition] > 1 {
+				t.Errorf("Partition %d assigned to multiple consumers", partition)
+			}
+		}
+		
+		// Each consumer should get exactly 1 partition (4 partitions / 4 consumers)
+		if len(assignment) != 1 {
+			t.Errorf("Consumer%d should get exactly 1 partition, got %d", i, len(assignment))
+		}
+	}
+
+	if totalAssigned != 4 {
+		t.Errorf("Expected 4 total partitions assigned, got %d", totalAssigned)
+	}
+
+	// Verify all partitions 0-3 are assigned
+	for i := int32(0); i < 4; i++ {
+		if assignedPartitions[i] != 1 {
+			t.Errorf("Partition %d assigned %d times, expected 1", i, assignedPartitions[i])
+		}
+	}
+
+	t.Logf("Multiple consumers join test completed successfully")
+}
+
+// RebalanceTestHandler implements sarama.ConsumerGroupHandler with rebalancing awareness
+type RebalanceTestHandler struct {
+	messages    chan *sarama.ConsumerMessage
+	ready       chan bool
+	assignments chan []int32
+	readyOnce   sync.Once
+	t           *testing.T
+	name        string
+}
+
+func (h *RebalanceTestHandler) Setup(session sarama.ConsumerGroupSession) error {
+	h.t.Logf("%s: Consumer group session setup", h.name)
+	h.readyOnce.Do(func() {
+		close(h.ready)
+	})
+	
+	// Send partition assignment
+	partitions := make([]int32, 0)
+	for topic, partitionList := range session.Claims() {
+		h.t.Logf("%s: Assigned topic %s with partitions %v", h.name, topic, partitionList)
+		for _, partition := range partitionList {
+			partitions = append(partitions, partition)
+		}
+	}
+	
+	select {
+	case h.assignments <- partitions:
+	default:
+		// Channel might be full, that's ok
+	}
+	
+	return nil
+}
+
+func (h *RebalanceTestHandler) Cleanup(sarama.ConsumerGroupSession) error {
+	h.t.Logf("%s: Consumer group session cleanup", h.name)
+	return nil
+}
+
+func (h *RebalanceTestHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
+	for {
+		select {
+		case message := <-claim.Messages():
+			if message == nil {
+				return nil
+			}
+			h.t.Logf("%s: Received message from partition %d: %s", h.name, message.Partition, string(message.Value))
+			select {
+			case h.messages <- message:
+			default:
+				// Channel full, drop message for test
+			}
+			session.MarkMessage(message, "")
+		case <-session.Context().Done():
+			return nil
+		}
+	}
+}

test/kafka/integration/schema_end_to_end_test.go

@@ -0,0 +1,299 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/linkedin/goavro/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/schema"
+)
+
+// TestSchemaEndToEnd_AvroRoundTrip tests the complete Avro schema round-trip workflow
+func TestSchemaEndToEnd_AvroRoundTrip(t *testing.T) {
+	// Create mock schema registry
+	server := createMockSchemaRegistryForE2E(t)
+	defer server.Close()
+
+	// Create schema manager
+	config := schema.ManagerConfig{
+		RegistryURL:    server.URL,
+		ValidationMode: schema.ValidationPermissive,
+	}
+	manager, err := schema.NewManager(config)
+	require.NoError(t, err)
+
+	// Test data
+	avroSchema := getUserAvroSchemaForE2E()
+	testData := map[string]interface{}{
+		"id":    int32(12345),
+		"name":  "Alice Johnson",
+		"email": map[string]interface{}{"string": "alice@example.com"}, // Avro union
+		"age":   map[string]interface{}{"int": int32(28)},              // Avro union
+		"preferences": map[string]interface{}{
+			"Preferences": map[string]interface{}{ // Avro union with record type
+				"notifications": true,
+				"theme":         "dark",
+			},
+		},
+	}
+
+	t.Run("SchemaManagerRoundTrip", func(t *testing.T) {
+		// Step 1: Create Confluent envelope (simulate producer)
+		codec, err := goavro.NewCodec(avroSchema)
+		require.NoError(t, err)
+
+		avroBinary, err := codec.BinaryFromNative(nil, testData)
+		require.NoError(t, err)
+
+		confluentMsg := schema.CreateConfluentEnvelope(schema.FormatAvro, 1, nil, avroBinary)
+		require.True(t, len(confluentMsg) > 0, "Confluent envelope should not be empty")
+
+		t.Logf("Created Confluent envelope: %d bytes", len(confluentMsg))
+
+		// Step 2: Decode message using schema manager
+		decodedMsg, err := manager.DecodeMessage(confluentMsg)
+		require.NoError(t, err)
+		require.NotNil(t, decodedMsg.RecordValue, "RecordValue should not be nil")
+
+		t.Logf("Decoded message with schema ID %d, format %v", decodedMsg.SchemaID, decodedMsg.SchemaFormat)
+
+		// Step 3: Re-encode message using schema manager
+		reconstructedMsg, err := manager.EncodeMessage(decodedMsg.RecordValue, 1, schema.FormatAvro)
+		require.NoError(t, err)
+		require.True(t, len(reconstructedMsg) > 0, "Reconstructed message should not be empty")
+
+		t.Logf("Re-encoded message: %d bytes", len(reconstructedMsg))
+
+		// Step 4: Verify the reconstructed message is a valid Confluent envelope
+		envelope, ok := schema.ParseConfluentEnvelope(reconstructedMsg)
+		require.True(t, ok, "Reconstructed message should be a valid Confluent envelope")
+		require.Equal(t, uint32(1), envelope.SchemaID, "Schema ID should match")
+		require.Equal(t, schema.FormatAvro, envelope.Format, "Schema format should be Avro")
+
+		// Step 5: Decode and verify the content
+		decodedNative, _, err := codec.NativeFromBinary(envelope.Payload)
+		require.NoError(t, err)
+
+		decodedMap, ok := decodedNative.(map[string]interface{})
+		require.True(t, ok, "Decoded data should be a map")
+
+		// Verify all fields
+		assert.Equal(t, int32(12345), decodedMap["id"])
+		assert.Equal(t, "Alice Johnson", decodedMap["name"])
+		
+		// Verify union fields
+		emailUnion, ok := decodedMap["email"].(map[string]interface{})
+		require.True(t, ok, "Email should be a union")
+		assert.Equal(t, "alice@example.com", emailUnion["string"])
+
+		ageUnion, ok := decodedMap["age"].(map[string]interface{})
+		require.True(t, ok, "Age should be a union")
+		assert.Equal(t, int32(28), ageUnion["int"])
+
+		preferencesUnion, ok := decodedMap["preferences"].(map[string]interface{})
+		require.True(t, ok, "Preferences should be a union")
+		preferencesRecord, ok := preferencesUnion["Preferences"].(map[string]interface{})
+		require.True(t, ok, "Preferences should contain a record")
+		assert.Equal(t, true, preferencesRecord["notifications"])
+		assert.Equal(t, "dark", preferencesRecord["theme"])
+
+		t.Log("Successfully completed Avro schema round-trip test")
+	})
+}
+
+// TestSchemaEndToEnd_ProtobufRoundTrip tests the complete Protobuf schema round-trip workflow
+func TestSchemaEndToEnd_ProtobufRoundTrip(t *testing.T) {
+	t.Run("ProtobufEnvelopeCreation", func(t *testing.T) {
+		// Create a simple Protobuf message (simulated)
+		// In a real scenario, this would be generated from a .proto file
+		protobufData := []byte{0x08, 0x96, 0x01, 0x12, 0x04, 0x74, 0x65, 0x73, 0x74} // id=150, name="test"
+
+		// Create Confluent envelope with Protobuf format
+		confluentMsg := schema.CreateConfluentEnvelope(schema.FormatProtobuf, 2, []int{0}, protobufData)
+		require.True(t, len(confluentMsg) > 0, "Confluent envelope should not be empty")
+
+		t.Logf("Created Protobuf Confluent envelope: %d bytes", len(confluentMsg))
+
+		// Verify Confluent envelope
+		envelope, ok := schema.ParseConfluentEnvelope(confluentMsg)
+		require.True(t, ok, "Message should be a valid Confluent envelope")
+		require.Equal(t, uint32(2), envelope.SchemaID, "Schema ID should match")
+		// Note: ParseConfluentEnvelope defaults to FormatAvro; format detection requires schema registry
+		require.Equal(t, schema.FormatAvro, envelope.Format, "Format defaults to Avro without schema registry lookup")
+		
+		// For Protobuf with indexes, we need to use the specialized parser
+		protobufEnvelope, ok := schema.ParseConfluentProtobufEnvelopeWithIndexCount(confluentMsg, 1)
+		require.True(t, ok, "Message should be a valid Protobuf envelope")
+		require.Equal(t, uint32(2), protobufEnvelope.SchemaID, "Schema ID should match")
+		require.Equal(t, schema.FormatProtobuf, protobufEnvelope.Format, "Schema format should be Protobuf")
+		require.Equal(t, []int{0}, protobufEnvelope.Indexes, "Indexes should match")
+		require.Equal(t, protobufData, protobufEnvelope.Payload, "Payload should match")
+
+		t.Log("Successfully completed Protobuf envelope test")
+	})
+}
+
+// TestSchemaEndToEnd_JSONSchemaRoundTrip tests the complete JSON Schema round-trip workflow
+func TestSchemaEndToEnd_JSONSchemaRoundTrip(t *testing.T) {
+	t.Run("JSONSchemaEnvelopeCreation", func(t *testing.T) {
+		// Create JSON data
+		jsonData := []byte(`{"id": 123, "name": "Bob Smith", "active": true}`)
+
+		// Create Confluent envelope with JSON Schema format
+		confluentMsg := schema.CreateConfluentEnvelope(schema.FormatJSONSchema, 3, nil, jsonData)
+		require.True(t, len(confluentMsg) > 0, "Confluent envelope should not be empty")
+
+		t.Logf("Created JSON Schema Confluent envelope: %d bytes", len(confluentMsg))
+
+		// Verify Confluent envelope
+		envelope, ok := schema.ParseConfluentEnvelope(confluentMsg)
+		require.True(t, ok, "Message should be a valid Confluent envelope")
+		require.Equal(t, uint32(3), envelope.SchemaID, "Schema ID should match")
+		// Note: ParseConfluentEnvelope defaults to FormatAvro; format detection requires schema registry
+		require.Equal(t, schema.FormatAvro, envelope.Format, "Format defaults to Avro without schema registry lookup")
+
+		// Verify JSON content
+		assert.JSONEq(t, string(jsonData), string(envelope.Payload), "JSON payload should match")
+
+		t.Log("Successfully completed JSON Schema envelope test")
+	})
+}
+
+// TestSchemaEndToEnd_CompressionAndBatching tests schema handling with compression and batching
+func TestSchemaEndToEnd_CompressionAndBatching(t *testing.T) {
+	// Create mock schema registry
+	server := createMockSchemaRegistryForE2E(t)
+	defer server.Close()
+
+	// Create schema manager
+	config := schema.ManagerConfig{
+		RegistryURL:    server.URL,
+		ValidationMode: schema.ValidationPermissive,
+	}
+	manager, err := schema.NewManager(config)
+	require.NoError(t, err)
+
+	t.Run("BatchedSchematizedMessages", func(t *testing.T) {
+		// Create multiple messages
+		avroSchema := getUserAvroSchemaForE2E()
+		codec, err := goavro.NewCodec(avroSchema)
+		require.NoError(t, err)
+
+		messageCount := 5
+		var confluentMessages [][]byte
+
+		// Create multiple Confluent envelopes
+		for i := 0; i < messageCount; i++ {
+			testData := map[string]interface{}{
+				"id":    int32(1000 + i),
+				"name":  fmt.Sprintf("User %d", i),
+				"email": map[string]interface{}{"string": fmt.Sprintf("user%d@example.com", i)},
+				"age":   map[string]interface{}{"int": int32(20 + i)},
+				"preferences": map[string]interface{}{
+					"Preferences": map[string]interface{}{
+						"notifications": i%2 == 0, // Alternate true/false
+						"theme":         "light",
+					},
+				},
+			}
+
+			avroBinary, err := codec.BinaryFromNative(nil, testData)
+			require.NoError(t, err)
+
+			confluentMsg := schema.CreateConfluentEnvelope(schema.FormatAvro, 1, nil, avroBinary)
+			confluentMessages = append(confluentMessages, confluentMsg)
+		}
+
+		t.Logf("Created %d schematized messages", messageCount)
+
+		// Test round-trip for each message
+		for i, confluentMsg := range confluentMessages {
+			// Decode message
+			decodedMsg, err := manager.DecodeMessage(confluentMsg)
+			require.NoError(t, err, "Message %d should decode", i)
+
+			// Re-encode message
+			reconstructedMsg, err := manager.EncodeMessage(decodedMsg.RecordValue, 1, schema.FormatAvro)
+			require.NoError(t, err, "Message %d should re-encode", i)
+
+			// Verify envelope
+			envelope, ok := schema.ParseConfluentEnvelope(reconstructedMsg)
+			require.True(t, ok, "Message %d should be a valid Confluent envelope", i)
+			require.Equal(t, uint32(1), envelope.SchemaID, "Message %d schema ID should match", i)
+
+			// Decode and verify content
+			decodedNative, _, err := codec.NativeFromBinary(envelope.Payload)
+			require.NoError(t, err, "Message %d should decode successfully", i)
+
+			decodedMap, ok := decodedNative.(map[string]interface{})
+			require.True(t, ok, "Message %d should be a map", i)
+
+			expectedID := int32(1000 + i)
+			assert.Equal(t, expectedID, decodedMap["id"], "Message %d ID should match", i)
+			assert.Equal(t, fmt.Sprintf("User %d", i), decodedMap["name"], "Message %d name should match", i)
+		}
+
+		t.Log("Successfully verified batched schematized messages")
+	})
+}
+
+// Helper functions for creating mock schema registries
+
+func createMockSchemaRegistryForE2E(t *testing.T) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/schemas/ids/1":
+			response := map[string]interface{}{
+				"schema":  getUserAvroSchemaForE2E(),
+				"subject": "user-events-e2e-value",
+				"version": 1,
+			}
+			writeJSONResponse(w, response)
+		case "/subjects/user-events-e2e-value/versions/latest":
+			response := map[string]interface{}{
+				"id":      1,
+				"schema":  getUserAvroSchemaForE2E(),
+				"subject": "user-events-e2e-value",
+				"version": 1,
+			}
+			writeJSONResponse(w, response)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+}
+
+
+func getUserAvroSchemaForE2E() string {
+	return `{
+		"type": "record",
+		"name": "User",
+		"fields": [
+			{"name": "id", "type": "int"},
+			{"name": "name", "type": "string"},
+			{"name": "email", "type": ["null", "string"], "default": null},
+			{"name": "age", "type": ["null", "int"], "default": null},
+			{"name": "preferences", "type": ["null", {
+				"type": "record",
+				"name": "Preferences",
+				"fields": [
+					{"name": "notifications", "type": "boolean", "default": true},
+					{"name": "theme", "type": "string", "default": "light"}
+				]
+			}], "default": null}
+		]
+	}`
+}
+
+func writeJSONResponse(w http.ResponseWriter, data interface{}) {
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(data); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}

test/kafka/integration/schema_registry_test.go

@@ -0,0 +1,210 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+// TestSchemaRegistryEventualConsistency reproduces the issue where schemas
+// are registered successfully but are not immediately queryable due to
+// Schema Registry's consumer lag
+func TestSchemaRegistryEventualConsistency(t *testing.T) {
+	// This test requires real SMQ backend
+	gateway := testutil.NewGatewayTestServerWithSMQ(t, testutil.SMQRequired)
+	defer gateway.CleanupAndClose()
+
+	addr := gateway.StartAndWait()
+	t.Logf("Gateway running on %s", addr)
+
+	// Schema Registry URL from environment or default
+	schemaRegistryURL := "http://localhost:8081"
+
+	// Wait for Schema Registry to be ready
+	if !waitForSchemaRegistry(t, schemaRegistryURL, 30*time.Second) {
+		t.Fatal("Schema Registry not ready")
+	}
+
+	// Define test schemas
+	valueSchema := `{"type":"record","name":"TestMessage","fields":[{"name":"id","type":"string"}]}`
+	keySchema := `{"type":"string"}`
+
+	// Register multiple schemas rapidly (simulates the load test scenario)
+	subjects := []string{
+		"test-topic-0-value",
+		"test-topic-0-key",
+		"test-topic-1-value",
+		"test-topic-1-key",
+		"test-topic-2-value",
+		"test-topic-2-key",
+		"test-topic-3-value",
+		"test-topic-3-key",
+	}
+
+	t.Log("Registering schemas rapidly...")
+	registeredIDs := make(map[string]int)
+	for _, subject := range subjects {
+		schema := valueSchema
+		if strings.HasSuffix(subject, "-key") {
+			schema = keySchema
+		}
+
+		id, err := registerSchema(schemaRegistryURL, subject, schema)
+		if err != nil {
+			t.Fatalf("Failed to register schema for %s: %v", subject, err)
+		}
+		registeredIDs[subject] = id
+		t.Logf("Registered %s with ID %d", subject, id)
+	}
+
+	t.Log("All schemas registered successfully!")
+
+	// Now immediately try to verify them (this reproduces the bug)
+	t.Log("Immediately verifying schemas (without delay)...")
+	immediateFailures := 0
+	for _, subject := range subjects {
+		exists, id, version, err := verifySchema(schemaRegistryURL, subject)
+		if err != nil || !exists {
+			immediateFailures++
+			t.Logf("Immediate verification failed for %s: exists=%v id=%d err=%v", subject, exists, id, err)
+		} else {
+			t.Logf("Immediate verification passed for %s: ID=%d Version=%d", subject, id, version)
+		}
+	}
+
+	if immediateFailures > 0 {
+		t.Logf("BUG REPRODUCED: %d/%d schemas not immediately queryable after registration",
+			immediateFailures, len(subjects))
+		t.Logf("  This is due to Schema Registry's KafkaStoreReaderThread lag")
+	}
+
+	// Now verify with retry logic (this should succeed)
+	t.Log("Verifying schemas with retry logic...")
+	for _, subject := range subjects {
+		expectedID := registeredIDs[subject]
+		if !verifySchemaWithRetry(t, schemaRegistryURL, subject, expectedID, 5*time.Second) {
+			t.Errorf("Failed to verify %s even with retry", subject)
+		}
+	}
+
+	t.Log("✓ All schemas verified successfully with retry logic!")
+}
+
+// registerSchema registers a schema and returns its ID
+func registerSchema(registryURL, subject, schema string) (int, error) {
+	// Escape the schema JSON
+	escapedSchema, err := json.Marshal(schema)
+	if err != nil {
+		return 0, err
+	}
+
+	payload := fmt.Sprintf(`{"schema":%s,"schemaType":"AVRO"}`, escapedSchema)
+
+	resp, err := http.Post(
+		fmt.Sprintf("%s/subjects/%s/versions", registryURL, subject),
+		"application/vnd.schemaregistry.v1+json",
+		strings.NewReader(payload),
+	)
+	if err != nil {
+		return 0, err
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("registration failed: %s - %s", resp.Status, string(body))
+	}
+
+	var result struct {
+		ID int `json:"id"`
+	}
+	if err := json.Unmarshal(body, &result); err != nil {
+		return 0, err
+	}
+
+	return result.ID, nil
+}
+
+// verifySchema checks if a schema exists
+func verifySchema(registryURL, subject string) (exists bool, id int, version int, err error) {
+	resp, err := http.Get(fmt.Sprintf("%s/subjects/%s/versions/latest", registryURL, subject))
+	if err != nil {
+		return false, 0, 0, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotFound {
+		return false, 0, 0, nil
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return false, 0, 0, fmt.Errorf("verification failed: %s - %s", resp.Status, string(body))
+	}
+
+	var result struct {
+		ID      int    `json:"id"`
+		Version int    `json:"version"`
+		Schema  string `json:"schema"`
+	}
+	body, _ := io.ReadAll(resp.Body)
+	if err := json.Unmarshal(body, &result); err != nil {
+		return false, 0, 0, err
+	}
+
+	return true, result.ID, result.Version, nil
+}
+
+// verifySchemaWithRetry verifies a schema with retry logic
+func verifySchemaWithRetry(t *testing.T, registryURL, subject string, expectedID int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	attempt := 0
+
+	for time.Now().Before(deadline) {
+		attempt++
+		exists, id, version, err := verifySchema(registryURL, subject)
+
+		if err == nil && exists && id == expectedID {
+			if attempt > 1 {
+				t.Logf("✓ %s verified after %d attempts (ID=%d, Version=%d)", subject, attempt, id, version)
+			}
+			return true
+		}
+
+		// Wait before retry (exponential backoff)
+		waitTime := time.Duration(attempt*100) * time.Millisecond
+		if waitTime > 1*time.Second {
+			waitTime = 1 * time.Second
+		}
+		time.Sleep(waitTime)
+	}
+
+	t.Logf("%s verification timed out after %d attempts", subject, attempt)
+	return false
+}
+
+// waitForSchemaRegistry waits for Schema Registry to be ready
+func waitForSchemaRegistry(t *testing.T, url string, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+
+	for time.Now().Before(deadline) {
+		resp, err := http.Get(url + "/subjects")
+		if err == nil && resp.StatusCode == http.StatusOK {
+			resp.Body.Close()
+			return true
+		}
+		if resp != nil {
+			resp.Body.Close()
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	return false
+}

test/kafka/integration/smq_integration_test.go

@@ -0,0 +1,305 @@
+package integration
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/IBM/sarama"
+	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
+)
+
+// TestSMQIntegration tests that the Kafka gateway properly integrates with SeaweedMQ
+// This test REQUIRES SeaweedFS masters to be running and will skip if not available
+func TestSMQIntegration(t *testing.T) {
+	// This test requires SMQ to be available
+	gateway := testutil.NewGatewayTestServerWithSMQ(t, testutil.SMQRequired)
+	defer gateway.CleanupAndClose()
+
+	addr := gateway.StartAndWait()
+
+	t.Logf("Running SMQ integration test with SeaweedFS backend")
+
+	t.Run("ProduceConsumeWithPersistence", func(t *testing.T) {
+		testProduceConsumeWithPersistence(t, addr)
+	})
+
+	t.Run("ConsumerGroupOffsetPersistence", func(t *testing.T) {
+		testConsumerGroupOffsetPersistence(t, addr)
+	})
+
+	t.Run("TopicPersistence", func(t *testing.T) {
+		testTopicPersistence(t, addr)
+	})
+}
+
+func testProduceConsumeWithPersistence(t *testing.T, addr string) {
+	topicName := testutil.GenerateUniqueTopicName("smq-integration-produce-consume")
+
+	client := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Create topic
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	// Allow time for topic to propagate in SMQ backend
+	time.Sleep(500 * time.Millisecond)
+
+	// Produce messages
+	messages := msgGen.GenerateStringMessages(5)
+	err = client.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages")
+
+	// Allow time for messages to be fully persisted in SMQ backend
+	time.Sleep(200 * time.Millisecond)
+
+	t.Logf("Produced %d messages to topic %s", len(messages), topicName)
+
+	// Consume messages
+	consumed, err := client.ConsumeMessages(topicName, 0, len(messages))
+	testutil.AssertNoError(t, err, "Failed to consume messages")
+
+	// Verify all messages were consumed
+	testutil.AssertEqual(t, len(messages), len(consumed), "Message count mismatch")
+
+	t.Logf("Successfully consumed %d messages from SMQ backend", len(consumed))
+}
+
+func testConsumerGroupOffsetPersistence(t *testing.T, addr string) {
+	topicName := testutil.GenerateUniqueTopicName("smq-integration-offset-persistence")
+	groupID := testutil.GenerateUniqueGroupID("smq-offset-group")
+
+	client := testutil.NewSaramaClient(t, addr)
+	msgGen := testutil.NewMessageGenerator()
+
+	// Create topic and produce messages
+	err := client.CreateTopic(topicName, 1, 1)
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	// Allow time for topic to propagate in SMQ backend
+	time.Sleep(500 * time.Millisecond)
+
+	messages := msgGen.GenerateStringMessages(10)
+	err = client.ProduceMessages(topicName, messages)
+	testutil.AssertNoError(t, err, "Failed to produce messages")
+
+	// Allow time for messages to be fully persisted in SMQ backend
+	time.Sleep(200 * time.Millisecond)
+
+	// Phase 1: Consume first 5 messages with consumer group and commit offsets
+	t.Logf("Phase 1: Consuming first 5 messages and committing offsets")
+
+	config := client.GetConfig()
+	config.Consumer.Offsets.Initial = sarama.OffsetOldest
+	// Enable auto-commit for more reliable offset handling
+	config.Consumer.Offsets.AutoCommit.Enable = true
+	config.Consumer.Offsets.AutoCommit.Interval = 1 * time.Second
+
+	consumerGroup1, err := sarama.NewConsumerGroup([]string{addr}, groupID, config)
+	testutil.AssertNoError(t, err, "Failed to create first consumer group")
+
+	handler := &SMQOffsetTestHandler{
+		messages:  make(chan *sarama.ConsumerMessage, len(messages)),
+		ready:     make(chan bool),
+		stopAfter: 5,
+		t:         t,
+	}
+
+	ctx1, cancel1 := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel1()
+
+	consumeErrChan1 := make(chan error, 1)
+	go func() {
+		err := consumerGroup1.Consume(ctx1, []string{topicName}, handler)
+		if err != nil && err != context.DeadlineExceeded && err != context.Canceled {
+			t.Logf("First consumer error: %v", err)
+			consumeErrChan1 <- err
+		}
+	}()
+
+	// Wait for consumer to be ready with timeout
+	select {
+	case <-handler.ready:
+		// Consumer is ready, continue
+	case err := <-consumeErrChan1:
+		t.Fatalf("First consumer failed to start: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatalf("Timeout waiting for first consumer to be ready")
+	}
+	consumedCount := 0
+	for consumedCount < 5 {
+		select {
+		case <-handler.messages:
+			consumedCount++
+		case <-time.After(20 * time.Second):
+			t.Fatalf("Timeout waiting for first batch of messages. Got %d/5", consumedCount)
+		}
+	}
+
+	consumerGroup1.Close()
+	cancel1()
+	time.Sleep(7 * time.Second) // Allow auto-commit to complete and offset commits to be processed in SMQ
+
+	t.Logf("Consumed %d messages in first phase", consumedCount)
+
+	// Phase 2: Start new consumer group with same ID - should resume from committed offset
+	t.Logf("Phase 2: Starting new consumer group to test offset persistence")
+
+	// Create a fresh config for the second consumer group to avoid any state issues
+	config2 := client.GetConfig()
+	config2.Consumer.Offsets.Initial = sarama.OffsetOldest
+	config2.Consumer.Offsets.AutoCommit.Enable = true
+	config2.Consumer.Offsets.AutoCommit.Interval = 1 * time.Second
+
+	consumerGroup2, err := sarama.NewConsumerGroup([]string{addr}, groupID, config2)
+	testutil.AssertNoError(t, err, "Failed to create second consumer group")
+	defer consumerGroup2.Close()
+
+	handler2 := &SMQOffsetTestHandler{
+		messages:  make(chan *sarama.ConsumerMessage, len(messages)),
+		ready:     make(chan bool),
+		stopAfter: 5, // Should consume remaining 5 messages
+		t:         t,
+	}
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel2()
+
+	consumeErrChan := make(chan error, 1)
+	go func() {
+		err := consumerGroup2.Consume(ctx2, []string{topicName}, handler2)
+		if err != nil && err != context.DeadlineExceeded && err != context.Canceled {
+			t.Logf("Second consumer error: %v", err)
+			consumeErrChan <- err
+		}
+	}()
+
+	// Wait for second consumer to be ready with timeout
+	select {
+	case <-handler2.ready:
+		// Consumer is ready, continue
+	case err := <-consumeErrChan:
+		t.Fatalf("Second consumer failed to start: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatalf("Timeout waiting for second consumer to be ready")
+	}
+	secondConsumerMessages := make([]*sarama.ConsumerMessage, 0)
+	consumedCount = 0
+	for consumedCount < 5 {
+		select {
+		case msg := <-handler2.messages:
+			consumedCount++
+			secondConsumerMessages = append(secondConsumerMessages, msg)
+		case <-time.After(20 * time.Second):
+			t.Fatalf("Timeout waiting for second batch of messages. Got %d/5", consumedCount)
+		}
+	}
+
+	// Verify second consumer started from correct offset (should be >= 5)
+	if len(secondConsumerMessages) > 0 {
+		firstMessageOffset := secondConsumerMessages[0].Offset
+		if firstMessageOffset < 5 {
+			t.Fatalf("Second consumer should start from offset >= 5: got %d", firstMessageOffset)
+		}
+		t.Logf("Second consumer correctly resumed from offset %d", firstMessageOffset)
+	}
+
+	t.Logf("Successfully verified SMQ offset persistence")
+}
+
+func testTopicPersistence(t *testing.T, addr string) {
+	topicName := testutil.GenerateUniqueTopicName("smq-integration-topic-persistence")
+
+	client := testutil.NewSaramaClient(t, addr)
+
+	// Create topic
+	err := client.CreateTopic(topicName, 2, 1) // 2 partitions
+	testutil.AssertNoError(t, err, "Failed to create topic")
+
+	// Allow time for topic to propagate and persist in SMQ backend
+	time.Sleep(1 * time.Second)
+
+	// Verify topic exists by listing topics using admin client
+	config := client.GetConfig()
+	config.Admin.Timeout = 30 * time.Second
+
+	admin, err := sarama.NewClusterAdmin([]string{addr}, config)
+	testutil.AssertNoError(t, err, "Failed to create admin client")
+	defer admin.Close()
+
+	// Retry topic listing to handle potential delays in topic propagation
+	var topics map[string]sarama.TopicDetail
+	var listErr error
+	for attempt := 0; attempt < 3; attempt++ {
+		if attempt > 0 {
+			sleepDuration := time.Duration(500*(1<<(attempt-1))) * time.Millisecond
+			t.Logf("Retrying ListTopics after %v (attempt %d/3)", sleepDuration, attempt+1)
+			time.Sleep(sleepDuration)
+		}
+
+		topics, listErr = admin.ListTopics()
+		if listErr == nil {
+			break
+		}
+	}
+	testutil.AssertNoError(t, listErr, "Failed to list topics")
+
+	topicDetails, exists := topics[topicName]
+	if !exists {
+		t.Fatalf("Topic %s not found in topic list", topicName)
+	}
+
+	if topicDetails.NumPartitions != 2 {
+		t.Errorf("Expected 2 partitions, got %d", topicDetails.NumPartitions)
+	}
+
+	t.Logf("Successfully verified topic persistence with %d partitions", topicDetails.NumPartitions)
+}
+
+// SMQOffsetTestHandler implements sarama.ConsumerGroupHandler for SMQ offset testing
+type SMQOffsetTestHandler struct {
+	messages  chan *sarama.ConsumerMessage
+	ready     chan bool
+	readyOnce bool
+	stopAfter int
+	consumed  int
+	t         *testing.T
+}
+
+func (h *SMQOffsetTestHandler) Setup(sarama.ConsumerGroupSession) error {
+	h.t.Logf("SMQ offset test consumer setup")
+	if !h.readyOnce {
+		close(h.ready)
+		h.readyOnce = true
+	}
+	return nil
+}
+
+func (h *SMQOffsetTestHandler) Cleanup(sarama.ConsumerGroupSession) error {
+	h.t.Logf("SMQ offset test consumer cleanup")
+	return nil
+}
+
+func (h *SMQOffsetTestHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
+	for {
+		select {
+		case message := <-claim.Messages():
+			if message == nil {
+				return nil
+			}
+			h.consumed++
+			h.messages <- message
+			session.MarkMessage(message, "")
+
+			// Stop after consuming the specified number of messages
+			if h.consumed >= h.stopAfter {
+				h.t.Logf("Stopping SMQ consumer after %d messages", h.consumed)
+				// Auto-commit will handle offset commits automatically
+				return nil
+			}
+		case <-session.Context().Done():
+			return nil
+		}
+	}
+}

test/kafka/internal/testutil/assertions.go

@@ -0,0 +1,150 @@
+package testutil
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+// AssertEventually retries an assertion until it passes or times out
+func AssertEventually(t *testing.T, assertion func() error, timeout time.Duration, interval time.Duration, msgAndArgs ...interface{}) {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+
+	for time.Now().Before(deadline) {
+		if err := assertion(); err == nil {
+			return // Success
+		} else {
+			lastErr = err
+		}
+		time.Sleep(interval)
+	}
+
+	// Format the failure message
+	var msg string
+	if len(msgAndArgs) > 0 {
+		if format, ok := msgAndArgs[0].(string); ok {
+			msg = fmt.Sprintf(format, msgAndArgs[1:]...)
+		} else {
+			msg = fmt.Sprint(msgAndArgs...)
+		}
+	} else {
+		msg = "assertion failed"
+	}
+
+	t.Fatalf("%s after %v: %v", msg, timeout, lastErr)
+}
+
+// AssertNoError fails the test if err is not nil
+func AssertNoError(t *testing.T, err error, msgAndArgs ...interface{}) {
+	t.Helper()
+	if err != nil {
+		var msg string
+		if len(msgAndArgs) > 0 {
+			if format, ok := msgAndArgs[0].(string); ok {
+				msg = fmt.Sprintf(format, msgAndArgs[1:]...)
+			} else {
+				msg = fmt.Sprint(msgAndArgs...)
+			}
+		} else {
+			msg = "unexpected error"
+		}
+		t.Fatalf("%s: %v", msg, err)
+	}
+}
+
+// AssertError fails the test if err is nil
+func AssertError(t *testing.T, err error, msgAndArgs ...interface{}) {
+	t.Helper()
+	if err == nil {
+		var msg string
+		if len(msgAndArgs) > 0 {
+			if format, ok := msgAndArgs[0].(string); ok {
+				msg = fmt.Sprintf(format, msgAndArgs[1:]...)
+			} else {
+				msg = fmt.Sprint(msgAndArgs...)
+			}
+		} else {
+			msg = "expected error but got nil"
+		}
+		t.Fatal(msg)
+	}
+}
+
+// AssertEqual fails the test if expected != actual
+func AssertEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) {
+	t.Helper()
+	if expected != actual {
+		var msg string
+		if len(msgAndArgs) > 0 {
+			if format, ok := msgAndArgs[0].(string); ok {
+				msg = fmt.Sprintf(format, msgAndArgs[1:]...)
+			} else {
+				msg = fmt.Sprint(msgAndArgs...)
+			}
+		} else {
+			msg = "values not equal"
+		}
+		t.Fatalf("%s: expected %v, got %v", msg, expected, actual)
+	}
+}
+
+// AssertNotEqual fails the test if expected == actual
+func AssertNotEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) {
+	t.Helper()
+	if expected == actual {
+		var msg string
+		if len(msgAndArgs) > 0 {
+			if format, ok := msgAndArgs[0].(string); ok {
+				msg = fmt.Sprintf(format, msgAndArgs[1:]...)
+			} else {
+				msg = fmt.Sprint(msgAndArgs...)
+			}
+		} else {
+			msg = "values should not be equal"
+		}
+		t.Fatalf("%s: both values are %v", msg, expected)
+	}
+}
+
+// AssertGreaterThan fails the test if actual <= expected
+func AssertGreaterThan(t *testing.T, expected, actual int, msgAndArgs ...interface{}) {
+	t.Helper()
+	if actual <= expected {
+		var msg string
+		if len(msgAndArgs) > 0 {
+			if format, ok := msgAndArgs[0].(string); ok {
+				msg = fmt.Sprintf(format, msgAndArgs[1:]...)
+			} else {
+				msg = fmt.Sprint(msgAndArgs...)
+			}
+		} else {
+			msg = "value not greater than expected"
+		}
+		t.Fatalf("%s: expected > %d, got %d", msg, expected, actual)
+	}
+}
+
+// AssertContains fails the test if slice doesn't contain item
+func AssertContains(t *testing.T, slice []string, item string, msgAndArgs ...interface{}) {
+	t.Helper()
+	for _, s := range slice {
+		if s == item {
+			return // Found it
+		}
+	}
+
+	var msg string
+	if len(msgAndArgs) > 0 {
+		if format, ok := msgAndArgs[0].(string); ok {
+			msg = fmt.Sprintf(format, msgAndArgs[1:]...)
+		} else {
+			msg = fmt.Sprint(msgAndArgs...)
+		}
+	} else {
+		msg = "item not found in slice"
+	}
+	t.Fatalf("%s: %q not found in %v", msg, item, slice)
+}

test/kafka/internal/testutil/clients.go

@@ -0,0 +1,305 @@
+package testutil
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/IBM/sarama"
+	"github.com/segmentio/kafka-go"
+)
+
+// KafkaGoClient wraps kafka-go client with test utilities
+type KafkaGoClient struct {
+	brokerAddr string
+	t          *testing.T
+}
+
+// SaramaClient wraps Sarama client with test utilities
+type SaramaClient struct {
+	brokerAddr string
+	config     *sarama.Config
+	t          *testing.T
+}
+
+// NewKafkaGoClient creates a new kafka-go test client
+func NewKafkaGoClient(t *testing.T, brokerAddr string) *KafkaGoClient {
+	return &KafkaGoClient{
+		brokerAddr: brokerAddr,
+		t:          t,
+	}
+}
+
+// NewSaramaClient creates a new Sarama test client with default config
+func NewSaramaClient(t *testing.T, brokerAddr string) *SaramaClient {
+	config := sarama.NewConfig()
+	config.Version = sarama.V2_8_0_0
+	config.Producer.Return.Successes = true
+	config.Consumer.Return.Errors = true
+	config.Consumer.Offsets.Initial = sarama.OffsetOldest // Start from earliest when no committed offset
+
+	return &SaramaClient{
+		brokerAddr: brokerAddr,
+		config:     config,
+		t:          t,
+	}
+}
+
+// CreateTopic creates a topic using kafka-go
+func (k *KafkaGoClient) CreateTopic(topicName string, partitions int, replicationFactor int) error {
+	k.t.Helper()
+
+	conn, err := kafka.Dial("tcp", k.brokerAddr)
+	if err != nil {
+		return fmt.Errorf("dial broker: %w", err)
+	}
+	defer conn.Close()
+
+	topicConfig := kafka.TopicConfig{
+		Topic:             topicName,
+		NumPartitions:     partitions,
+		ReplicationFactor: replicationFactor,
+	}
+
+	err = conn.CreateTopics(topicConfig)
+	if err != nil {
+		return fmt.Errorf("create topic: %w", err)
+	}
+
+	k.t.Logf("Created topic %s with %d partitions", topicName, partitions)
+	return nil
+}
+
+// ProduceMessages produces messages using kafka-go
+func (k *KafkaGoClient) ProduceMessages(topicName string, messages []kafka.Message) error {
+	k.t.Helper()
+
+	writer := &kafka.Writer{
+		Addr:         kafka.TCP(k.brokerAddr),
+		Topic:        topicName,
+		Balancer:     &kafka.LeastBytes{},
+		BatchTimeout: 50 * time.Millisecond,
+		RequiredAcks: kafka.RequireOne,
+	}
+	defer writer.Close()
+
+	// Increased timeout to handle slow CI environments, especially when consumer groups
+	// are active and holding locks or requiring offset commits
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	err := writer.WriteMessages(ctx, messages...)
+	if err != nil {
+		return fmt.Errorf("write messages: %w", err)
+	}
+
+	k.t.Logf("Produced %d messages to topic %s", len(messages), topicName)
+	return nil
+}
+
+// ConsumeMessages consumes messages using kafka-go
+func (k *KafkaGoClient) ConsumeMessages(topicName string, expectedCount int) ([]kafka.Message, error) {
+	k.t.Helper()
+
+	reader := kafka.NewReader(kafka.ReaderConfig{
+		Brokers:     []string{k.brokerAddr},
+		Topic:       topicName,
+		Partition:   0, // Explicitly set partition 0 for simple consumption
+		StartOffset: kafka.FirstOffset,
+		MinBytes:    1,
+		MaxBytes:    10e6,
+	})
+	defer reader.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	var messages []kafka.Message
+	for i := 0; i < expectedCount; i++ {
+		msg, err := reader.ReadMessage(ctx)
+		if err != nil {
+			return messages, fmt.Errorf("read message %d: %w", i, err)
+		}
+		messages = append(messages, msg)
+	}
+
+	k.t.Logf("Consumed %d messages from topic %s", len(messages), topicName)
+	return messages, nil
+}
+
+// ConsumeWithGroup consumes messages using consumer group
+func (k *KafkaGoClient) ConsumeWithGroup(topicName, groupID string, expectedCount int) ([]kafka.Message, error) {
+	k.t.Helper()
+
+	reader := kafka.NewReader(kafka.ReaderConfig{
+		Brokers:        []string{k.brokerAddr},
+		Topic:          topicName,
+		GroupID:        groupID,
+		MinBytes:       1,
+		MaxBytes:       10e6,
+		CommitInterval: 500 * time.Millisecond,
+	})
+	defer reader.Close()
+
+	// Log the initial offset position
+	offset := reader.Offset()
+	k.t.Logf("Consumer group reader created for group %s, initial offset: %d", groupID, offset)
+
+	// Increased timeout for consumer groups - they require coordinator discovery,
+	// offset fetching, and offset commits which can be slow in CI environments
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	var messages []kafka.Message
+	for i := 0; i < expectedCount; i++ {
+		// Fetch then explicitly commit to better control commit timing
+		msg, err := reader.FetchMessage(ctx)
+		if err != nil {
+			return messages, fmt.Errorf("read message %d: %w", i, err)
+		}
+		messages = append(messages, msg)
+		k.t.Logf("  Fetched message %d: offset=%d, partition=%d", i, msg.Offset, msg.Partition)
+
+		// Commit with simple retry to handle transient connection churn
+		var commitErr error
+		for attempt := 0; attempt < 3; attempt++ {
+			commitErr = reader.CommitMessages(ctx, msg)
+			if commitErr == nil {
+				k.t.Logf("  Committed offset %d (attempt %d)", msg.Offset, attempt+1)
+				break
+			}
+			k.t.Logf("  Commit attempt %d failed for offset %d: %v", attempt+1, msg.Offset, commitErr)
+			// brief backoff
+			time.Sleep(time.Duration(50*(1<<attempt)) * time.Millisecond)
+		}
+		if commitErr != nil {
+			return messages, fmt.Errorf("committing message %d: %w", i, commitErr)
+		}
+	}
+
+	k.t.Logf("Consumed %d messages from topic %s with group %s", len(messages), topicName, groupID)
+	return messages, nil
+}
+
+// CreateTopic creates a topic using Sarama
+func (s *SaramaClient) CreateTopic(topicName string, partitions int32, replicationFactor int16) error {
+	s.t.Helper()
+
+	admin, err := sarama.NewClusterAdmin([]string{s.brokerAddr}, s.config)
+	if err != nil {
+		return fmt.Errorf("create admin client: %w", err)
+	}
+	defer admin.Close()
+
+	topicDetail := &sarama.TopicDetail{
+		NumPartitions:     partitions,
+		ReplicationFactor: replicationFactor,
+	}
+
+	err = admin.CreateTopic(topicName, topicDetail, false)
+	if err != nil {
+		return fmt.Errorf("create topic: %w", err)
+	}
+
+	s.t.Logf("Created topic %s with %d partitions", topicName, partitions)
+	return nil
+}
+
+// ProduceMessages produces messages using Sarama
+func (s *SaramaClient) ProduceMessages(topicName string, messages []string) error {
+	s.t.Helper()
+
+	producer, err := sarama.NewSyncProducer([]string{s.brokerAddr}, s.config)
+	if err != nil {
+		return fmt.Errorf("create producer: %w", err)
+	}
+	defer producer.Close()
+
+	for i, msgText := range messages {
+		msg := &sarama.ProducerMessage{
+			Topic: topicName,
+			Key:   sarama.StringEncoder(fmt.Sprintf("Test message %d", i)),
+			Value: sarama.StringEncoder(msgText),
+		}
+
+		partition, offset, err := producer.SendMessage(msg)
+		if err != nil {
+			return fmt.Errorf("send message %d: %w", i, err)
+		}
+
+		s.t.Logf("Produced message %d: partition=%d, offset=%d", i, partition, offset)
+	}
+
+	return nil
+}
+
+// ProduceMessageToPartition produces a single message to a specific partition using Sarama
+func (s *SaramaClient) ProduceMessageToPartition(topicName string, partition int32, message string) error {
+	s.t.Helper()
+
+	producer, err := sarama.NewSyncProducer([]string{s.brokerAddr}, s.config)
+	if err != nil {
+		return fmt.Errorf("create producer: %w", err)
+	}
+	defer producer.Close()
+
+	msg := &sarama.ProducerMessage{
+		Topic:     topicName,
+		Partition: partition,
+		Key:       sarama.StringEncoder(fmt.Sprintf("key-p%d", partition)),
+		Value:     sarama.StringEncoder(message),
+	}
+
+	actualPartition, offset, err := producer.SendMessage(msg)
+	if err != nil {
+		return fmt.Errorf("send message to partition %d: %w", partition, err)
+	}
+
+	s.t.Logf("Produced message to partition %d: actualPartition=%d, offset=%d", partition, actualPartition, offset)
+	return nil
+}
+
+// ConsumeMessages consumes messages using Sarama
+func (s *SaramaClient) ConsumeMessages(topicName string, partition int32, expectedCount int) ([]string, error) {
+	s.t.Helper()
+
+	consumer, err := sarama.NewConsumer([]string{s.brokerAddr}, s.config)
+	if err != nil {
+		return nil, fmt.Errorf("create consumer: %w", err)
+	}
+	defer consumer.Close()
+
+	partitionConsumer, err := consumer.ConsumePartition(topicName, partition, sarama.OffsetOldest)
+	if err != nil {
+		return nil, fmt.Errorf("create partition consumer: %w", err)
+	}
+	defer partitionConsumer.Close()
+
+	var messages []string
+	timeout := time.After(30 * time.Second)
+
+	for len(messages) < expectedCount {
+		select {
+		case msg := <-partitionConsumer.Messages():
+			messages = append(messages, string(msg.Value))
+		case err := <-partitionConsumer.Errors():
+			return messages, fmt.Errorf("consumer error: %w", err)
+		case <-timeout:
+			return messages, fmt.Errorf("timeout waiting for messages, got %d/%d", len(messages), expectedCount)
+		}
+	}
+
+	s.t.Logf("Consumed %d messages from topic %s", len(messages), topicName)
+	return messages, nil
+}
+
+// GetConfig returns the Sarama configuration
+func (s *SaramaClient) GetConfig() *sarama.Config {
+	return s.config
+}
+
+// SetConfig sets a custom Sarama configuration
+func (s *SaramaClient) SetConfig(config *sarama.Config) {
+	s.config = config
+}

test/kafka/internal/testutil/docker.go

@@ -0,0 +1,68 @@
+package testutil
+
+import (
+	"os"
+	"testing"
+)
+
+// DockerEnvironment provides utilities for Docker-based integration tests
+type DockerEnvironment struct {
+	KafkaBootstrap string
+	KafkaGateway   string
+	SchemaRegistry string
+	Available      bool
+}
+
+// NewDockerEnvironment creates a new Docker environment helper
+func NewDockerEnvironment(t *testing.T) *DockerEnvironment {
+	t.Helper()
+
+	env := &DockerEnvironment{
+		KafkaBootstrap: os.Getenv("KAFKA_BOOTSTRAP_SERVERS"),
+		KafkaGateway:   os.Getenv("KAFKA_GATEWAY_URL"),
+		SchemaRegistry: os.Getenv("SCHEMA_REGISTRY_URL"),
+	}
+
+	env.Available = env.KafkaBootstrap != ""
+
+	if env.Available {
+		t.Logf("Docker environment detected:")
+		t.Logf("  Kafka Bootstrap: %s", env.KafkaBootstrap)
+		t.Logf("  Kafka Gateway: %s", env.KafkaGateway)
+		t.Logf("  Schema Registry: %s", env.SchemaRegistry)
+	}
+
+	return env
+}
+
+// SkipIfNotAvailable skips the test if Docker environment is not available
+func (d *DockerEnvironment) SkipIfNotAvailable(t *testing.T) {
+	t.Helper()
+	if !d.Available {
+		t.Skip("Skipping Docker integration test - set KAFKA_BOOTSTRAP_SERVERS to run")
+	}
+}
+
+// RequireKafka ensures Kafka is available or skips the test
+func (d *DockerEnvironment) RequireKafka(t *testing.T) {
+	t.Helper()
+	if d.KafkaBootstrap == "" {
+		t.Skip("Kafka bootstrap servers not available")
+	}
+}
+
+// RequireGateway ensures Kafka Gateway is available or skips the test
+func (d *DockerEnvironment) RequireGateway(t *testing.T) {
+	t.Helper()
+	if d.KafkaGateway == "" {
+		t.Skip("Kafka Gateway not available")
+	}
+}
+
+// RequireSchemaRegistry ensures Schema Registry is available or skips the test
+func (d *DockerEnvironment) RequireSchemaRegistry(t *testing.T) {
+	t.Helper()
+	if d.SchemaRegistry == "" {
+		t.Skip("Schema Registry not available")
+	}
+}

test/kafka/internal/testutil/gateway.go

@@ -0,0 +1,220 @@
+package testutil
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/gateway"
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/schema"
+)
+
+// GatewayTestServer wraps the gateway server with common test utilities
+type GatewayTestServer struct {
+	*gateway.Server
+	t *testing.T
+}
+
+// GatewayOptions contains configuration for test gateway
+type GatewayOptions struct {
+	Listen        string
+	Masters       string
+	UseProduction bool
+	// Add more options as needed
+}
+
+// NewGatewayTestServer creates a new test gateway server with common setup
+func NewGatewayTestServer(t *testing.T, opts GatewayOptions) *GatewayTestServer {
+	if opts.Listen == "" {
+		opts.Listen = "127.0.0.1:0" // Use random port by default
+	}
+
+	// Allow switching to production gateway if requested (requires masters)
+	var srv *gateway.Server
+	if opts.UseProduction {
+		if opts.Masters == "" {
+			// Fallback to env variable for convenience in CI
+			if v := os.Getenv("SEAWEEDFS_MASTERS"); v != "" {
+				opts.Masters = v
+			} else {
+				opts.Masters = "localhost:9333"
+			}
+		}
+		srv = gateway.NewServer(gateway.Options{
+			Listen:  opts.Listen,
+			Masters: opts.Masters,
+		})
+	} else {
+		// For unit testing without real SeaweedMQ masters
+		srv = gateway.NewTestServerForUnitTests(gateway.Options{
+			Listen: opts.Listen,
+		})
+	}
+
+	return &GatewayTestServer{
+		Server: srv,
+		t:      t,
+	}
+}
+
+// StartAndWait starts the gateway and waits for it to be ready
+func (g *GatewayTestServer) StartAndWait() string {
+	g.t.Helper()
+
+	// Start server in goroutine
+	go func() {
+		// Enable schema mode automatically when SCHEMA_REGISTRY_URL is set
+		if url := os.Getenv("SCHEMA_REGISTRY_URL"); url != "" {
+			h := g.GetHandler()
+			if h != nil {
+				_ = h.EnableSchemaManagement(schema.ManagerConfig{RegistryURL: url})
+			}
+		}
+		if err := g.Start(); err != nil {
+			g.t.Errorf("Failed to start gateway: %v", err)
+		}
+	}()
+
+	// Wait for server to be ready
+	time.Sleep(100 * time.Millisecond)
+
+	host, port := g.GetListenerAddr()
+	addr := fmt.Sprintf("%s:%d", host, port)
+	g.t.Logf("Gateway running on %s", addr)
+
+	return addr
+}
+
+// AddTestTopic adds a topic for testing with default configuration
+func (g *GatewayTestServer) AddTestTopic(name string) {
+	g.t.Helper()
+	g.GetHandler().AddTopicForTesting(name, 1)
+	g.t.Logf("Added test topic: %s", name)
+}
+
+// AddTestTopics adds multiple topics for testing
+func (g *GatewayTestServer) AddTestTopics(names ...string) {
+	g.t.Helper()
+	for _, name := range names {
+		g.AddTestTopic(name)
+	}
+}
+
+// CleanupAndClose properly closes the gateway server
+func (g *GatewayTestServer) CleanupAndClose() {
+	g.t.Helper()
+	if err := g.Close(); err != nil {
+		g.t.Errorf("Failed to close gateway: %v", err)
+	}
+}
+
+// SMQAvailabilityMode indicates whether SeaweedMQ is available for testing
+type SMQAvailabilityMode int
+
+const (
+	SMQUnavailable SMQAvailabilityMode = iota // Use mock handler only
+	SMQAvailable                              // SMQ is available, can use production mode
+	SMQRequired                               // SMQ is required, skip test if unavailable
+)
+
+// CheckSMQAvailability checks if SeaweedFS masters are available for testing
+func CheckSMQAvailability() (bool, string) {
+	masters := os.Getenv("SEAWEEDFS_MASTERS")
+	if masters == "" {
+		return false, ""
+	}
+
+	// Test if at least one master is reachable
+	if masters != "" {
+		// Try to connect to the first master to verify availability
+		conn, err := net.DialTimeout("tcp", masters, 2*time.Second)
+		if err != nil {
+			return false, masters // Masters specified but unreachable
+		}
+		conn.Close()
+		return true, masters
+	}
+
+	return false, ""
+}
+
+// NewGatewayTestServerWithSMQ creates a gateway server that automatically uses SMQ if available
+func NewGatewayTestServerWithSMQ(t *testing.T, mode SMQAvailabilityMode) *GatewayTestServer {
+	smqAvailable, masters := CheckSMQAvailability()
+
+	switch mode {
+	case SMQRequired:
+		if !smqAvailable {
+			if masters != "" {
+				t.Skipf("Skipping test: SEAWEEDFS_MASTERS=%s specified but unreachable", masters)
+			} else {
+				t.Skip("Skipping test: SEAWEEDFS_MASTERS required but not set")
+			}
+		}
+		t.Logf("Using SMQ-backed gateway with masters: %s", masters)
+		return newGatewayTestServerWithTimeout(t, GatewayOptions{
+			UseProduction: true,
+			Masters:       masters,
+		}, 120*time.Second)
+
+	case SMQAvailable:
+		if smqAvailable {
+			t.Logf("SMQ available, using production gateway with masters: %s", masters)
+			return newGatewayTestServerWithTimeout(t, GatewayOptions{
+				UseProduction: true,
+				Masters:       masters,
+			}, 120*time.Second)
+		} else {
+			t.Logf("SMQ not available, using mock gateway")
+			return NewGatewayTestServer(t, GatewayOptions{})
+		}
+
+	default: // SMQUnavailable
+		t.Logf("Using mock gateway (SMQ integration disabled)")
+		return NewGatewayTestServer(t, GatewayOptions{})
+	}
+}
+
+// newGatewayTestServerWithTimeout creates a gateway server with a timeout to prevent hanging
+func newGatewayTestServerWithTimeout(t *testing.T, opts GatewayOptions, timeout time.Duration) *GatewayTestServer {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	done := make(chan *GatewayTestServer, 1)
+	errChan := make(chan error, 1)
+
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				errChan <- fmt.Errorf("panic creating gateway: %v", r)
+			}
+		}()
+
+		// Create the gateway in a goroutine so we can timeout if it hangs
+		t.Logf("Creating gateway with masters: %s (with %v timeout)", opts.Masters, timeout)
+		gateway := NewGatewayTestServer(t, opts)
+		t.Logf("Gateway created successfully")
+		done <- gateway
+	}()
+
+	select {
+	case gateway := <-done:
+		return gateway
+	case err := <-errChan:
+		t.Fatalf("Error creating gateway: %v", err)
+	case <-ctx.Done():
+		t.Fatalf("Timeout creating gateway after %v - likely SMQ broker discovery failed. Check if MQ brokers are running and accessible.", timeout)
+	}
+
+	return nil // This should never be reached
+}
+
+// IsSMQMode returns true if the gateway is using real SMQ backend
+// This is determined by checking if we have the SEAWEEDFS_MASTERS environment variable
+func (g *GatewayTestServer) IsSMQMode() bool {
+	available, _ := CheckSMQAvailability()
+	return available
+}

test/kafka/internal/testutil/messages.go

@@ -0,0 +1,135 @@
+package testutil
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/schema"
+	"github.com/segmentio/kafka-go"
+)
+
+// MessageGenerator provides utilities for generating test messages
+type MessageGenerator struct {
+	counter int
+}
+
+// NewMessageGenerator creates a new message generator
+func NewMessageGenerator() *MessageGenerator {
+	return &MessageGenerator{counter: 0}
+}
+
+// GenerateKafkaGoMessages generates kafka-go messages for testing
+func (m *MessageGenerator) GenerateKafkaGoMessages(count int) []kafka.Message {
+	messages := make([]kafka.Message, count)
+
+	for i := 0; i < count; i++ {
+		m.counter++
+		key := []byte(fmt.Sprintf("test-key-%d", m.counter))
+		val := []byte(fmt.Sprintf("{\"value\":\"test-message-%d-generated-at-%d\"}", m.counter, time.Now().Unix()))
+
+		// If schema mode is requested, ensure a test schema exists and wrap with Confluent envelope
+		if url := os.Getenv("SCHEMA_REGISTRY_URL"); url != "" {
+			subject := "offset-management-value"
+			schemaJSON := `{"type":"record","name":"TestRecord","fields":[{"name":"value","type":"string"}]}`
+			rc := schema.NewRegistryClient(schema.RegistryConfig{URL: url})
+			if _, err := rc.GetLatestSchema(subject); err != nil {
+				// Best-effort register schema
+				_, _ = rc.RegisterSchema(subject, schemaJSON)
+			}
+			if latest, err := rc.GetLatestSchema(subject); err == nil {
+				val = schema.CreateConfluentEnvelope(schema.FormatAvro, latest.LatestID, nil, val)
+			} else {
+				// fallback to schema id 1
+				val = schema.CreateConfluentEnvelope(schema.FormatAvro, 1, nil, val)
+			}
+		}
+
+		messages[i] = kafka.Message{Key: key, Value: val}
+	}
+
+	return messages
+}
+
+// GenerateStringMessages generates string messages for Sarama
+func (m *MessageGenerator) GenerateStringMessages(count int) []string {
+	messages := make([]string, count)
+
+	for i := 0; i < count; i++ {
+		m.counter++
+		messages[i] = fmt.Sprintf("test-message-%d-generated-at-%d", m.counter, time.Now().Unix())
+	}
+
+	return messages
+}
+
+// GenerateKafkaGoMessage generates a single kafka-go message
+func (m *MessageGenerator) GenerateKafkaGoMessage(key, value string) kafka.Message {
+	if key == "" {
+		m.counter++
+		key = fmt.Sprintf("test-key-%d", m.counter)
+	}
+	if value == "" {
+		value = fmt.Sprintf("test-message-%d-generated-at-%d", m.counter, time.Now().Unix())
+	}
+
+	return kafka.Message{
+		Key:   []byte(key),
+		Value: []byte(value),
+	}
+}
+
+// GenerateUniqueTopicName generates a unique topic name for testing
+func GenerateUniqueTopicName(prefix string) string {
+	if prefix == "" {
+		prefix = "test-topic"
+	}
+	return fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())
+}
+
+// GenerateUniqueGroupID generates a unique consumer group ID for testing
+func GenerateUniqueGroupID(prefix string) string {
+	if prefix == "" {
+		prefix = "test-group"
+	}
+	return fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())
+}
+
+// ValidateMessageContent validates that consumed messages match expected content
+func ValidateMessageContent(expected, actual []string) error {
+	if len(expected) != len(actual) {
+		return fmt.Errorf("message count mismatch: expected %d, got %d", len(expected), len(actual))
+	}
+
+	for i, expectedMsg := range expected {
+		if i >= len(actual) {
+			return fmt.Errorf("missing message at index %d", i)
+		}
+		if actual[i] != expectedMsg {
+			return fmt.Errorf("message mismatch at index %d: expected %q, got %q", i, expectedMsg, actual[i])
+		}
+	}
+
+	return nil
+}
+
+// ValidateKafkaGoMessageContent validates kafka-go messages
+func ValidateKafkaGoMessageContent(expected, actual []kafka.Message) error {
+	if len(expected) != len(actual) {
+		return fmt.Errorf("message count mismatch: expected %d, got %d", len(expected), len(actual))
+	}
+
+	for i, expectedMsg := range expected {
+		if i >= len(actual) {
+			return fmt.Errorf("missing message at index %d", i)
+		}
+		if string(actual[i].Key) != string(expectedMsg.Key) {
+			return fmt.Errorf("key mismatch at index %d: expected %q, got %q", i, string(expectedMsg.Key), string(actual[i].Key))
+		}
+		if string(actual[i].Value) != string(expectedMsg.Value) {
+			return fmt.Errorf("value mismatch at index %d: expected %q, got %q", i, string(expectedMsg.Value), string(actual[i].Value))
+		}
+	}
+
+	return nil
+}

test/kafka/internal/testutil/schema_helper.go

@@ -0,0 +1,33 @@
+package testutil
+
+import (
+	"testing"
+
+	kschema "github.com/seaweedfs/seaweedfs/weed/mq/kafka/schema"
+)
+
+// EnsureValueSchema registers a minimal Avro value schema for the given topic if not present.
+// Returns the latest schema ID if successful.
+func EnsureValueSchema(t *testing.T, registryURL, topic string) (uint32, error) {
+	t.Helper()
+	subject := topic + "-value"
+	rc := kschema.NewRegistryClient(kschema.RegistryConfig{URL: registryURL})
+
+	// Minimal Avro record schema with string field "value"
+	schemaJSON := `{"type":"record","name":"TestRecord","fields":[{"name":"value","type":"string"}]}`
+
+	// Try to get existing
+	if latest, err := rc.GetLatestSchema(subject); err == nil {
+		return latest.LatestID, nil
+	}
+
+	// Register and fetch latest
+	if _, err := rc.RegisterSchema(subject, schemaJSON); err != nil {
+		return 0, err
+	}
+	latest, err := rc.GetLatestSchema(subject)
+	if err != nil {
+		return 0, err
+	}
+	return latest.LatestID, nil
+}

test/kafka/kafka-client-loadtest/.dockerignore

@@ -0,0 +1,3 @@
+# Keep only the Linux binaries
+!weed-linux-amd64
+!weed-linux-arm64

test/kafka/kafka-client-loadtest/.gitignore

@@ -0,0 +1,63 @@
+# Binaries
+kafka-loadtest
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool
+*.out
+
+# Go workspace file
+go.work
+
+# Test results and logs
+test-results/
+*.log
+logs/
+
+# Docker volumes and data
+data/
+volumes/
+
+# Monitoring data
+monitoring/prometheus/data/
+monitoring/grafana/data/
+
+# IDE files
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Environment files
+.env
+.env.local
+.env.*.local
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+
+# Coverage reports
+coverage.html
+coverage.out
+
+# Build artifacts
+bin/
+build/
+dist/

test/kafka/kafka-client-loadtest/Dockerfile.loadtest

@@ -0,0 +1,49 @@
+# Kafka Client Load Test Runner Dockerfile
+# Multi-stage build for cross-platform support
+
+# Stage 1: Builder
+FROM golang:1.24-alpine AS builder
+
+WORKDIR /app
+
+# Copy go module files
+COPY test/kafka/kafka-client-loadtest/go.mod test/kafka/kafka-client-loadtest/go.sum ./
+RUN go mod download
+
+# Copy source code
+COPY test/kafka/kafka-client-loadtest/ ./
+
+# Build the loadtest binary
+RUN CGO_ENABLED=0 GOOS=linux go build -o /kafka-loadtest ./cmd/loadtest
+
+# Stage 2: Runtime
+FROM ubuntu:22.04
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    curl \
+    jq \
+    bash \
+    netcat \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy built binary from builder stage
+COPY --from=builder /kafka-loadtest /usr/local/bin/kafka-loadtest
+RUN chmod +x /usr/local/bin/kafka-loadtest
+
+# Copy scripts and configuration
+COPY test/kafka/kafka-client-loadtest/scripts/ /scripts/
+COPY test/kafka/kafka-client-loadtest/config/ /config/
+
+# Create results directory
+RUN mkdir -p /test-results
+
+# Make scripts executable
+RUN chmod +x /scripts/*.sh
+
+WORKDIR /app
+
+# Default command runs the comprehensive load test
+CMD ["/usr/local/bin/kafka-loadtest", "-config", "/config/loadtest.yaml"]
+

test/kafka/kafka-client-loadtest/Dockerfile.seaweedfs

@@ -0,0 +1,37 @@
+# SeaweedFS Runtime Dockerfile for Kafka Client Load Tests
+# Optimized for fast builds - binary built locally and copied in
+FROM alpine:3.18
+
+# Install runtime dependencies
+RUN apk add --no-cache \
+    ca-certificates \
+    wget \
+    netcat-openbsd \
+    curl \
+    tzdata \
+    && rm -rf /var/cache/apk/*
+
+# Copy pre-built SeaweedFS binary (built locally for linux/amd64 or linux/arm64)
+# Cache-busting: Use build arg to force layer rebuild on every build
+ARG TARGETARCH=arm64
+ARG CACHE_BUST=unknown
+RUN echo "Building with cache bust: ${CACHE_BUST}"
+COPY weed-linux-${TARGETARCH} /usr/local/bin/weed
+RUN chmod +x /usr/local/bin/weed
+
+# Create data directory
+RUN mkdir -p /data
+
+# Set timezone
+ENV TZ=UTC
+
+# Health check script
+RUN echo '#!/bin/sh' > /usr/local/bin/health-check && \
+    echo 'exec "$@"' >> /usr/local/bin/health-check && \
+    chmod +x /usr/local/bin/health-check
+
+VOLUME ["/data"]
+WORKDIR /data
+
+ENTRYPOINT ["/usr/local/bin/weed"]
+

test/kafka/kafka-client-loadtest/Dockerfile.seektest

@@ -0,0 +1,20 @@
+FROM openjdk:11-jdk-slim
+
+# Install Maven
+RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Create source directory
+RUN mkdir -p src/main/java
+
+# Copy source and build files
+COPY SeekToBeginningTest.java src/main/java/
+COPY pom.xml .
+
+# Compile and package
+RUN mvn clean package -DskipTests
+
+# Run the test
+ENTRYPOINT ["java", "-cp", "target/seek-test.jar", "SeekToBeginningTest"]
+CMD ["kafka-gateway:9093"]

test/kafka/kafka-client-loadtest/Makefile

@@ -0,0 +1,446 @@
+# Kafka Client Load Test Makefile
+# Provides convenient targets for running load tests against SeaweedFS Kafka Gateway
+
+.PHONY: help build start stop restart clean test quick-test stress-test endurance-test monitor logs status
+
+# Configuration
+DOCKER_COMPOSE := docker compose
+PROJECT_NAME := kafka-client-loadtest
+CONFIG_FILE := config/loadtest.yaml
+
+# Build configuration
+GOARCH ?= arm64
+GOOS ?= linux
+
+# Default test parameters
+TEST_MODE ?= comprehensive
+TEST_DURATION ?= 300s
+PRODUCER_COUNT ?= 10
+CONSUMER_COUNT ?= 5
+MESSAGE_RATE ?= 1000
+MESSAGE_SIZE ?= 1024
+
+# Colors for output
+GREEN := \033[0;32m
+YELLOW := \033[0;33m
+BLUE := \033[0;34m
+NC := \033[0m
+
+help: ## Show this help message
+	@echo "Kafka Client Load Test Makefile"
+	@echo ""
+	@echo "Available targets:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  $(BLUE)%-20s$(NC) %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+	@echo ""
+	@echo "Environment variables:"
+	@echo "  TEST_MODE       Test mode: producer, consumer, comprehensive (default: comprehensive)"
+	@echo "  TEST_DURATION   Test duration (default: 300s)"
+	@echo "  PRODUCER_COUNT  Number of producers (default: 10)"
+	@echo "  CONSUMER_COUNT  Number of consumers (default: 5)"
+	@echo "  MESSAGE_RATE    Messages per second per producer (default: 1000)"
+	@echo "  MESSAGE_SIZE    Message size in bytes (default: 1024)"
+	@echo ""
+	@echo "Examples:"
+	@echo "  make test                              # Run default comprehensive test"
+	@echo "  make test TEST_DURATION=10m           # Run 10-minute test"
+	@echo "  make quick-test                        # Run quick smoke test (rebuilds gateway)"
+	@echo "  make stress-test                       # Run high-load stress test"
+	@echo "  make test TEST_MODE=producer           # Producer-only test"
+	@echo "  make schema-test                       # Run schema integration test with Schema Registry"
+	@echo "  make schema-quick-test                 # Run quick schema test (30s timeout)"
+	@echo "  make schema-loadtest                   # Run load test with schemas enabled"
+	@echo "  make build-binary                      # Build SeaweedFS binary locally for Linux"
+	@echo "  make build-gateway                     # Build Kafka Gateway (builds binary + Docker image)"
+	@echo "  make build-gateway-clean               # Build Kafka Gateway with no cache (fresh build)"
+
+build: ## Build the load test application
+	@echo "$(BLUE)Building load test application...$(NC)"
+	$(DOCKER_COMPOSE) build kafka-client-loadtest
+	@echo "$(GREEN)Build completed$(NC)"
+
+build-binary: ## Build the SeaweedFS binary locally for Linux
+	@echo "$(BLUE)Building SeaweedFS binary locally for $(GOOS) $(GOARCH)...$(NC)"
+	cd ../../.. && \
+	CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build \
+		-ldflags="-s -w" \
+		-tags "5BytesOffset" \
+		-o test/kafka/kafka-client-loadtest/weed-$(GOOS)-$(GOARCH) \
+		weed/weed.go
+	@echo "$(GREEN)Binary build completed: weed-$(GOOS)-$(GOARCH)$(NC)"
+
+build-gateway: build-binary ## Build the Kafka Gateway with latest changes
+	@echo "$(BLUE)Building Kafka Gateway Docker image...$(NC)"
+	CACHE_BUST=$$(date +%s) $(DOCKER_COMPOSE) build kafka-gateway
+	@echo "$(GREEN)Kafka Gateway build completed$(NC)"
+
+build-gateway-clean: build-binary ## Build the Kafka Gateway with no cache (force fresh build)
+	@echo "$(BLUE)Building Kafka Gateway Docker image with no cache...$(NC)"
+	$(DOCKER_COMPOSE) build --no-cache kafka-gateway
+	@echo "$(GREEN)Kafka Gateway clean build completed$(NC)"
+
+setup: ## Set up monitoring and configuration
+	@echo "$(BLUE)Setting up monitoring configuration...$(NC)"
+	./scripts/setup-monitoring.sh
+	@echo "$(GREEN)Setup completed$(NC)"
+
+start: build-gateway ## Start the infrastructure services (without load test)
+	@echo "$(BLUE)Starting SeaweedFS infrastructure...$(NC)"
+	$(DOCKER_COMPOSE) up -d \
+		seaweedfs-master \
+		seaweedfs-volume \
+		seaweedfs-filer \
+		seaweedfs-mq-broker \
+		kafka-gateway \
+		schema-registry-init \
+		schema-registry
+	@echo "$(GREEN)Infrastructure started$(NC)"
+	@echo "Waiting for services to be ready..."
+	./scripts/wait-for-services.sh wait
+	@echo "$(GREEN)All services are ready!$(NC)"
+
+stop: ## Stop all services
+	@echo "$(BLUE)Stopping all services...$(NC)"
+	$(DOCKER_COMPOSE) --profile loadtest --profile monitoring down
+	@echo "$(GREEN)Services stopped$(NC)"
+
+restart: stop start ## Restart all services
+
+clean: ## Clean up all resources (containers, volumes, networks, local data)
+	@echo "$(YELLOW)Warning: This will remove all volumes and data!$(NC)"
+	@echo "Press Ctrl+C to cancel, or wait 5 seconds to continue..."
+	@sleep 5
+	@echo "$(BLUE)Cleaning up all resources...$(NC)"
+	$(DOCKER_COMPOSE) --profile loadtest --profile monitoring down -v --remove-orphans
+	docker system prune -f
+	@if [ -f "weed-linux-arm64" ]; then \
+		echo "$(BLUE)Removing local binary...$(NC)"; \
+		rm -f weed-linux-arm64; \
+	fi
+	@if [ -d "data" ]; then \
+		echo "$(BLUE)Removing ALL local data directories (including offset state)...$(NC)"; \
+		rm -rf data/*; \
+	fi
+	@echo "$(GREEN)Cleanup completed - all data removed$(NC)"
+
+clean-binary: ## Clean up only the local binary
+	@echo "$(BLUE)Removing local binary...$(NC)"
+	@rm -f weed-linux-arm64
+	@echo "$(GREEN)Binary cleanup completed$(NC)"
+
+status: ## Show service status
+	@echo "$(BLUE)Service Status:$(NC)"
+	$(DOCKER_COMPOSE) ps
+
+logs: ## Show logs from all services
+	$(DOCKER_COMPOSE) logs -f
+
+test: start ## Run the comprehensive load test
+	@echo "$(BLUE)Running Kafka client load test...$(NC)"
+	@echo "Mode: $(TEST_MODE), Duration: $(TEST_DURATION)"
+	@echo "Producers: $(PRODUCER_COUNT), Consumers: $(CONSUMER_COUNT)"
+	@echo "Message Rate: $(MESSAGE_RATE) msgs/sec, Size: $(MESSAGE_SIZE) bytes"
+	@echo ""
+	@docker rm -f kafka-client-loadtest-runner 2>/dev/null || true
+	TEST_MODE=$(TEST_MODE) TEST_DURATION=$(TEST_DURATION) PRODUCER_COUNT=$(PRODUCER_COUNT) CONSUMER_COUNT=$(CONSUMER_COUNT) MESSAGE_RATE=$(MESSAGE_RATE) MESSAGE_SIZE=$(MESSAGE_SIZE) VALUE_TYPE=$(VALUE_TYPE) $(DOCKER_COMPOSE) --profile loadtest up --abort-on-container-exit kafka-client-loadtest
+	@echo "$(GREEN)Load test completed!$(NC)"
+	@$(MAKE) show-results
+
+quick-test: build-gateway ## Run a quick smoke test (1 min, low load, WITH schemas)
+	@echo "$(BLUE)================================================================$(NC)"
+	@echo "$(BLUE)    Quick Test (Low Load, WITH Schema Registry + Avro)       $(NC)"
+	@echo "$(BLUE)  - Duration: 1 minute                                        $(NC)"
+	@echo "$(BLUE)  - Load: 1 producer × 10 msg/sec = 10 total msg/sec         $(NC)"
+	@echo "$(BLUE)  - Message Type: Avro (with schema encoding)                $(NC)"
+	@echo "$(BLUE)  - Schema-First: Registers schemas BEFORE producing         $(NC)"
+	@echo "$(BLUE)================================================================$(NC)"
+	@echo ""
+	@$(MAKE) start
+	@echo ""
+	@echo "$(BLUE)=== Step 1: Registering schemas in Schema Registry ===$(NC)"
+	@echo "$(YELLOW)[WARN] IMPORTANT: Schemas MUST be registered before producing Avro messages!$(NC)"
+	@./scripts/register-schemas.sh full
+	@echo "$(GREEN)- Schemas registered successfully$(NC)"
+	@echo ""
+	@echo "$(BLUE)=== Step 2: Running load test with Avro messages ===$(NC)"
+	@$(MAKE) test \
+		TEST_MODE=comprehensive \
+		TEST_DURATION=60s \
+		PRODUCER_COUNT=1 \
+		CONSUMER_COUNT=1 \
+		MESSAGE_RATE=10 \
+		MESSAGE_SIZE=256 \
+		VALUE_TYPE=avro
+	@echo ""
+	@echo "$(GREEN)================================================================$(NC)"
+	@echo "$(GREEN)                    Quick Test Complete!                      $(NC)"
+	@echo "$(GREEN)  - Schema Registration                                       $(NC)"
+	@echo "$(GREEN)  - Avro Message Production                                   $(NC)"
+	@echo "$(GREEN)  - Message Consumption                                       $(NC)"
+	@echo "$(GREEN)================================================================$(NC)"
+
+standard-test: ## Run a standard load test (2 min, medium load, WITH Schema Registry + Avro)
+	@echo "$(BLUE)================================================================$(NC)"
+	@echo "$(BLUE)      Standard Test (Medium Load, WITH Schema Registry)      $(NC)"
+	@echo "$(BLUE)  - Duration: 2 minutes                                       $(NC)"
+	@echo "$(BLUE)  - Load: 2 producers × 50 msg/sec = 100 total msg/sec       $(NC)"
+	@echo "$(BLUE)  - Message Type: Avro (with schema encoding)                $(NC)"
+	@echo "$(BLUE)  - IMPORTANT: Schemas registered FIRST in Schema Registry   $(NC)"
+	@echo "$(BLUE)================================================================$(NC)"
+	@echo ""
+	@$(MAKE) start
+	@echo ""
+	@echo "$(BLUE)=== Step 1: Registering schemas in Schema Registry ===$(NC)"
+	@echo "$(YELLOW)Note: Schemas MUST be registered before producing Avro messages!$(NC)"
+	@./scripts/register-schemas.sh full
+	@echo "$(GREEN)- Schemas registered$(NC)"
+	@echo ""
+	@echo "$(BLUE)=== Step 2: Running load test with Avro messages ===$(NC)"
+	@$(MAKE) test \
+		TEST_MODE=comprehensive \
+		TEST_DURATION=2m \
+		PRODUCER_COUNT=2 \
+		CONSUMER_COUNT=2 \
+		MESSAGE_RATE=50 \
+		MESSAGE_SIZE=512 \
+		VALUE_TYPE=avro
+	@echo ""
+	@echo "$(GREEN)================================================================$(NC)"
+	@echo "$(GREEN)                  Standard Test Complete!                     $(NC)"
+	@echo "$(GREEN)================================================================$(NC)"
+
+stress-test: ## Run a stress test (10 minutes, high load) with schemas
+	@echo "$(BLUE)Starting stress test with schema registration...$(NC)"
+	@$(MAKE) start
+	@echo "$(BLUE)Registering schemas with Schema Registry...$(NC)"
+	@./scripts/register-schemas.sh full
+	@echo "$(BLUE)Running stress test with registered schemas...$(NC)"
+	@$(MAKE) test \
+		TEST_MODE=comprehensive \
+		TEST_DURATION=10m \
+		PRODUCER_COUNT=20 \
+		CONSUMER_COUNT=10 \
+		MESSAGE_RATE=2000 \
+		MESSAGE_SIZE=2048 \
+		VALUE_TYPE=avro
+
+endurance-test: ## Run an endurance test (30 minutes, sustained load) with schemas
+	@echo "$(BLUE)Starting endurance test with schema registration...$(NC)"
+	@$(MAKE) start
+	@echo "$(BLUE)Registering schemas with Schema Registry...$(NC)"
+	@./scripts/register-schemas.sh full
+	@echo "$(BLUE)Running endurance test with registered schemas...$(NC)"
+	@$(MAKE) test \
+		TEST_MODE=comprehensive \
+		TEST_DURATION=30m \
+		PRODUCER_COUNT=10 \
+		CONSUMER_COUNT=5 \
+		MESSAGE_RATE=1000 \
+		MESSAGE_SIZE=1024 \
+		VALUE_TYPE=avro
+
+producer-test: ## Run producer-only load test
+	@$(MAKE) test TEST_MODE=producer
+
+consumer-test: ## Run consumer-only load test (requires existing messages)
+	@$(MAKE) test TEST_MODE=consumer
+
+register-schemas: start ## Register schemas with Schema Registry
+	@echo "$(BLUE)Registering schemas with Schema Registry...$(NC)"
+	@./scripts/register-schemas.sh full
+	@echo "$(GREEN)Schema registration completed!$(NC)"
+
+verify-schemas: ## Verify schemas are registered in Schema Registry
+	@echo "$(BLUE)Verifying schemas in Schema Registry...$(NC)"
+	@./scripts/register-schemas.sh verify
+	@echo "$(GREEN)Schema verification completed!$(NC)"
+
+list-schemas: ## List all registered schemas in Schema Registry
+	@echo "$(BLUE)Listing registered schemas...$(NC)"
+	@./scripts/register-schemas.sh list
+
+cleanup-schemas: ## Clean up test schemas from Schema Registry
+	@echo "$(YELLOW)Cleaning up test schemas...$(NC)"
+	@./scripts/register-schemas.sh cleanup
+	@echo "$(GREEN)Schema cleanup completed!$(NC)"
+
+schema-test: start ## Run schema integration test (with Schema Registry)
+	@echo "$(BLUE)Running schema integration test...$(NC)"
+	@echo "Testing Schema Registry integration with schematized topics"
+	@echo ""
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o schema-test-linux test_schema_integration.go
+	docker run --rm --network kafka-client-loadtest \
+		-v $(PWD)/schema-test-linux:/usr/local/bin/schema-test \
+		alpine:3.18 /usr/local/bin/schema-test
+	@rm -f schema-test-linux
+	@echo "$(GREEN)Schema integration test completed!$(NC)"
+
+schema-quick-test: start ## Run quick schema test (lighter version)
+	@echo "$(BLUE)Running quick schema test...$(NC)"
+	@echo "Testing basic schema functionality"
+	@echo ""
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o schema-test-linux test_schema_integration.go
+	timeout 60s docker run --rm --network kafka-client-loadtest \
+		-v $(PWD)/schema-test-linux:/usr/local/bin/schema-test \
+		alpine:3.18 /usr/local/bin/schema-test || true
+	@rm -f schema-test-linux
+	@echo "$(GREEN)Quick schema test completed!$(NC)"
+
+simple-schema-test: start ## Run simple schema test (step-by-step)
+	@echo "$(BLUE)Running simple schema test...$(NC)"
+	@echo "Step-by-step schema functionality test"
+	@echo ""
+	@mkdir -p simple-test
+	@cp simple_schema_test.go simple-test/main.go
+	cd simple-test && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../simple-schema-test-linux .
+	docker run --rm --network kafka-client-loadtest \
+		-v $(PWD)/simple-schema-test-linux:/usr/local/bin/simple-schema-test \
+		alpine:3.18 /usr/local/bin/simple-schema-test
+	@rm -f simple-schema-test-linux
+	@rm -rf simple-test
+	@echo "$(GREEN)Simple schema test completed!$(NC)"
+
+basic-schema-test: start ## Run basic schema test (manual schema handling without Schema Registry)
+	@echo "$(BLUE)Running basic schema test...$(NC)"
+	@echo "Testing schema functionality without Schema Registry dependency"
+	@echo ""
+	@mkdir -p basic-test
+	@cp basic_schema_test.go basic-test/main.go
+	cd basic-test && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../basic-schema-test-linux .
+	timeout 60s docker run --rm --network kafka-client-loadtest \
+		-v $(PWD)/basic-schema-test-linux:/usr/local/bin/basic-schema-test \
+		alpine:3.18 /usr/local/bin/basic-schema-test
+	@rm -f basic-schema-test-linux
+	@rm -rf basic-test
+	@echo "$(GREEN)Basic schema test completed!$(NC)"
+
+schema-loadtest: start ## Run load test with schemas enabled
+	@echo "$(BLUE)Running schema-enabled load test...$(NC)"
+	@echo "Mode: comprehensive with schemas, Duration: 3m"
+	@echo "Producers: 3, Consumers: 2, Message Rate: 50 msgs/sec"
+	@echo ""
+	TEST_MODE=comprehensive \
+	TEST_DURATION=3m \
+	PRODUCER_COUNT=3 \
+	CONSUMER_COUNT=2 \
+	MESSAGE_RATE=50 \
+	MESSAGE_SIZE=1024 \
+	SCHEMA_REGISTRY_URL=http://schema-registry:8081 \
+	$(DOCKER_COMPOSE) --profile loadtest up --abort-on-container-exit kafka-client-loadtest
+	@echo "$(GREEN)Schema load test completed!$(NC)"
+	@$(MAKE) show-results
+
+monitor: setup ## Start monitoring stack (Prometheus + Grafana)
+	@echo "$(BLUE)Starting monitoring stack...$(NC)"
+	$(DOCKER_COMPOSE) --profile monitoring up -d prometheus grafana
+	@echo "$(GREEN)Monitoring stack started!$(NC)"
+	@echo ""
+	@echo "Access points:"
+	@echo "  Prometheus: http://localhost:9090"
+	@echo "  Grafana:    http://localhost:3000 (admin/admin)"
+
+monitor-stop: ## Stop monitoring stack
+	@echo "$(BLUE)Stopping monitoring stack...$(NC)"
+	$(DOCKER_COMPOSE) --profile monitoring stop prometheus grafana
+	@echo "$(GREEN)Monitoring stack stopped$(NC)"
+
+test-with-monitoring: monitor start ## Run test with monitoring enabled
+	@echo "$(BLUE)Running load test with monitoring...$(NC)"
+	@$(MAKE) test
+	@echo ""
+	@echo "$(GREEN)Test completed! Check the monitoring dashboards:$(NC)"
+	@echo "  Prometheus: http://localhost:9090"
+	@echo "  Grafana:    http://localhost:3000 (admin/admin)"
+
+show-results: ## Show test results
+	@echo "$(BLUE)Test Results Summary:$(NC)"
+	@if $(DOCKER_COMPOSE) ps -q kafka-client-loadtest-runner >/dev/null 2>&1; then \
+		$(DOCKER_COMPOSE) exec -T kafka-client-loadtest-runner curl -s http://localhost:8080/stats 2>/dev/null || echo "Results not available"; \
+	else \
+		echo "Load test container not running"; \
+	fi
+	@echo ""
+	@if [ -d "test-results" ]; then \
+		echo "Detailed results saved to: test-results/"; \
+		ls -la test-results/ 2>/dev/null || true; \
+	fi
+
+health-check: ## Check health of all services
+	@echo "$(BLUE)Checking service health...$(NC)"
+	./scripts/wait-for-services.sh check
+
+validate-setup: ## Validate the test setup
+	@echo "$(BLUE)Validating test setup...$(NC)"
+	@echo "Checking Docker and Docker Compose..."
+	@docker --version
+	@docker compose version || docker-compose --version
+	@echo ""
+	@echo "Checking configuration file..."
+	@if [ -f "$(CONFIG_FILE)" ]; then \
+		echo "- Configuration file exists: $(CONFIG_FILE)"; \
+	else \
+		echo "x Configuration file not found: $(CONFIG_FILE)"; \
+		exit 1; \
+	fi
+	@echo ""
+	@echo "Checking scripts..."
+	@for script in scripts/*.sh; do \
+		if [ -x "$$script" ]; then \
+			echo "- $$script is executable"; \
+		else \
+			echo "x $$script is not executable"; \
+		fi; \
+	done
+	@echo "$(GREEN)Setup validation completed$(NC)"
+
+dev-env: ## Set up development environment
+	@echo "$(BLUE)Setting up development environment...$(NC)"
+	@echo "Installing Go dependencies..."
+	go mod download
+	go mod tidy
+	@echo "$(GREEN)Development environment ready$(NC)"
+
+benchmark: ## Run comprehensive benchmarking suite
+	@echo "$(BLUE)Running comprehensive benchmark suite...$(NC)"
+	@echo "This will run multiple test scenarios and collect detailed metrics"
+	@echo ""
+	@$(MAKE) quick-test
+	@sleep 10
+	@$(MAKE) standard-test  
+	@sleep 10
+	@$(MAKE) stress-test
+	@echo "$(GREEN)Benchmark suite completed!$(NC)"
+
+# Advanced targets
+debug: ## Start services in debug mode with verbose logging
+	@echo "$(BLUE)Starting services in debug mode...$(NC)"
+	SEAWEEDFS_LOG_LEVEL=debug \
+	KAFKA_LOG_LEVEL=debug \
+	$(DOCKER_COMPOSE) up \
+		seaweedfs-master \
+		seaweedfs-volume \
+		seaweedfs-filer \
+		seaweedfs-mq-broker \
+		kafka-gateway \
+		schema-registry
+
+attach-loadtest: ## Attach to running load test container
+	$(DOCKER_COMPOSE) exec kafka-client-loadtest-runner /bin/sh
+
+exec-master: ## Execute shell in SeaweedFS master container
+	$(DOCKER_COMPOSE) exec seaweedfs-master /bin/sh
+
+exec-filer: ## Execute shell in SeaweedFS filer container
+	$(DOCKER_COMPOSE) exec seaweedfs-filer /bin/sh
+
+exec-gateway: ## Execute shell in Kafka gateway container
+	$(DOCKER_COMPOSE) exec kafka-gateway /bin/sh
+
+# Utility targets
+ps: status ## Alias for status
+
+up: start ## Alias for start
+
+down: stop ## Alias for stop
+
+# Help is the default target
+.DEFAULT_GOAL := help

test/kafka/kafka-client-loadtest/README.md

@@ -0,0 +1,397 @@
+# Kafka Client Load Test for SeaweedFS
+
+This comprehensive load testing suite validates the SeaweedFS MQ stack using real Kafka client libraries. Unlike the existing SMQ tests, this uses actual Kafka clients (`sarama` and `confluent-kafka-go`) to test the complete integration through:
+
+- **Kafka Clients** → **SeaweedFS Kafka Gateway** → **SeaweedFS MQ Broker** → **SeaweedFS Storage**
+
+## Architecture
+
+```
+┌─────────────────┐    ┌──────────────────┐    ┌─────────────────────┐
+│   Kafka Client  │    │  Kafka Gateway   │    │   SeaweedFS MQ      │
+│   Load Test     │───▶│  (Port 9093)     │───▶│   Broker            │
+│   - Producers   │    │                  │    │                     │
+│   - Consumers   │    │  Protocol        │    │   Topic Management  │
+│                 │    │  Translation     │    │   Message Storage   │
+└─────────────────┘    └──────────────────┘    └─────────────────────┘
+                                                             │
+                                                             ▼
+                                                ┌─────────────────────┐
+                                                │  SeaweedFS Storage  │
+                                                │  - Master           │
+                                                │  - Volume Server    │
+                                                │  - Filer            │
+                                                └─────────────────────┘
+```
+
+## Features
+
+### 🚀 **Multiple Test Modes**
+- **Producer-only**: Pure message production testing
+- **Consumer-only**: Consumption from existing topics  
+- **Comprehensive**: Full producer + consumer load testing
+
+### 📊 **Rich Metrics & Monitoring**
+- Prometheus metrics collection
+- Grafana dashboards
+- Real-time throughput and latency tracking
+- Consumer lag monitoring
+- Error rate analysis
+
+### 🔧 **Configurable Test Scenarios**
+- **Quick Test**: 1-minute smoke test
+- **Standard Test**: 5-minute medium load
+- **Stress Test**: 10-minute high load  
+- **Endurance Test**: 30-minute sustained load
+- **Custom**: Fully configurable parameters
+
+### 📈 **Message Types**
+- **JSON**: Structured test messages
+- **Avro**: Schema Registry integration
+- **Binary**: Raw binary payloads
+
+### 🛠 **Kafka Client Support**
+- **Sarama**: Native Go Kafka client
+- **Confluent**: Official Confluent Go client
+- Schema Registry integration
+- Consumer group management
+
+## Quick Start
+
+### Prerequisites
+- Docker & Docker Compose
+- Make (optional, but recommended)
+
+### 1. Run Default Test
+```bash
+make test
+```
+This runs a 5-minute comprehensive test with 10 producers and 5 consumers.
+
+### 2. Quick Smoke Test
+```bash
+make quick-test
+```
+1-minute test with minimal load for validation.
+
+### 3. Stress Test
+```bash
+make stress-test  
+```
+10-minute high-throughput test with 20 producers and 10 consumers.
+
+### 4. Test with Monitoring
+```bash
+make test-with-monitoring
+```
+Includes Prometheus + Grafana dashboards for real-time monitoring.
+
+## Detailed Usage
+
+### Manual Control
+```bash
+# Start infrastructure only
+make start
+
+# Run load test against running infrastructure
+make test TEST_MODE=comprehensive TEST_DURATION=10m
+
+# Stop everything
+make stop
+
+# Clean up all resources
+make clean
+```
+
+### Using Scripts Directly
+```bash
+# Full control with the main script
+./scripts/run-loadtest.sh start -m comprehensive -d 10m --monitoring
+
+# Check service health
+./scripts/wait-for-services.sh check
+
+# Setup monitoring configurations
+./scripts/setup-monitoring.sh
+```
+
+### Environment Variables
+```bash
+export TEST_MODE=comprehensive        # producer, consumer, comprehensive  
+export TEST_DURATION=300s            # Test duration
+export PRODUCER_COUNT=10              # Number of producer instances
+export CONSUMER_COUNT=5               # Number of consumer instances  
+export MESSAGE_RATE=1000              # Messages/second per producer
+export MESSAGE_SIZE=1024              # Message size in bytes
+export TOPIC_COUNT=5                  # Number of topics to create
+export PARTITIONS_PER_TOPIC=3         # Partitions per topic
+
+make test
+```
+
+## Configuration
+
+### Main Configuration File
+Edit `config/loadtest.yaml` to customize:
+
+- **Kafka Settings**: Bootstrap servers, security, timeouts
+- **Producer Config**: Batching, compression, acknowledgments  
+- **Consumer Config**: Group settings, fetch parameters
+- **Message Settings**: Size, format (JSON/Avro/Binary)
+- **Schema Registry**: Avro/Protobuf schema validation
+- **Metrics**: Prometheus collection intervals
+- **Test Scenarios**: Predefined load patterns
+
+### Example Custom Configuration
+```yaml
+test_mode: "comprehensive"
+duration: "600s"  # 10 minutes
+
+producers:
+  count: 15
+  message_rate: 2000
+  message_size: 2048
+  compression_type: "snappy"
+  acks: "all"
+
+consumers:
+  count: 8
+  group_prefix: "high-load-group"
+  max_poll_records: 1000
+
+topics:
+  count: 10
+  partitions: 6
+  replication_factor: 1
+```
+
+## Test Scenarios
+
+### 1. Producer Performance Test
+```bash
+make producer-test TEST_DURATION=10m PRODUCER_COUNT=20 MESSAGE_RATE=3000
+```
+Tests maximum message production throughput.
+
+### 2. Consumer Performance Test  
+```bash
+# First produce messages
+make producer-test TEST_DURATION=5m
+
+# Then test consumption
+make consumer-test TEST_DURATION=10m CONSUMER_COUNT=15
+```
+
+### 3. Schema Registry Integration
+```bash
+# Enable schemas in config/loadtest.yaml
+schemas:
+  enabled: true
+  
+make test
+```
+Tests Avro message serialization through Schema Registry.
+
+### 4. High Availability Test
+```bash
+# Test with container restarts during load
+make test TEST_DURATION=20m &
+sleep 300
+docker restart kafka-gateway
+```
+
+## Monitoring & Metrics
+
+### Real-Time Dashboards
+When monitoring is enabled:
+- **Prometheus**: http://localhost:9090
+- **Grafana**: http://localhost:3000 (admin/admin)
+
+### Key Metrics Tracked
+- **Throughput**: Messages/second, MB/second
+- **Latency**: End-to-end message latency percentiles  
+- **Errors**: Producer/consumer error rates
+- **Consumer Lag**: Per-partition lag monitoring
+- **Resource Usage**: CPU, memory, disk I/O
+
+### Grafana Dashboards
+- **Kafka Load Test**: Comprehensive test metrics
+- **SeaweedFS Cluster**: Storage system health
+- **Custom Dashboards**: Extensible monitoring
+
+## Advanced Features
+
+### Schema Registry Testing
+```bash
+# Test Avro message serialization
+export KAFKA_VALUE_TYPE=avro
+make test
+```
+
+The load test includes:
+- Schema registration
+- Avro message encoding/decoding  
+- Schema evolution testing
+- Compatibility validation
+
+### Multi-Client Testing
+The test supports both Sarama and Confluent clients:
+```go
+// Configure in producer/consumer code
+useConfluent := true  // Switch client implementation
+```
+
+### Consumer Group Rebalancing
+- Automatic consumer group management
+- Partition rebalancing simulation
+- Consumer failure recovery testing
+
+### Chaos Testing
+```yaml
+chaos:
+  enabled: true
+  producer_failure_rate: 0.01
+  consumer_failure_rate: 0.01
+  network_partition_probability: 0.001
+```
+
+## Troubleshooting
+
+### Common Issues
+
+#### Services Not Starting
+```bash
+# Check service health
+make health-check
+
+# View detailed logs
+make logs
+
+# Debug mode
+make debug
+```
+
+#### Low Throughput
+- Increase `MESSAGE_RATE` and `PRODUCER_COUNT`
+- Adjust `batch_size` and `linger_ms` in config
+- Check consumer `max_poll_records` setting
+
+#### High Latency
+- Reduce `linger_ms` for lower latency
+- Adjust `acks` setting (0, 1, or "all")
+- Monitor consumer lag
+
+#### Memory Issues  
+```bash
+# Reduce concurrent clients
+make test PRODUCER_COUNT=5 CONSUMER_COUNT=3
+
+# Adjust message size  
+make test MESSAGE_SIZE=512
+```
+
+### Debug Commands
+```bash
+# Execute shell in containers
+make exec-master
+make exec-filer  
+make exec-gateway
+
+# Attach to load test
+make attach-loadtest
+
+# View real-time stats
+curl http://localhost:8080/stats
+```
+
+## Development
+
+### Building from Source
+```bash
+# Set up development environment
+make dev-env
+
+# Build load test binary
+make build
+
+# Run tests locally (requires Go 1.21+)
+cd cmd/loadtest && go run main.go -config ../../config/loadtest.yaml
+```
+
+### Extending the Tests
+1. **Add new message formats** in `internal/producer/`
+2. **Add custom metrics** in `internal/metrics/`  
+3. **Create new test scenarios** in `config/loadtest.yaml`
+4. **Add monitoring panels** in `monitoring/grafana/dashboards/`
+
+### Contributing
+1. Fork the repository
+2. Create a feature branch
+3. Add tests for new functionality
+4. Ensure all tests pass: `make test`
+5. Submit a pull request
+
+## Performance Benchmarks
+
+### Expected Performance (on typical hardware)
+
+| Scenario | Producers | Consumers | Rate (msg/s) | Latency (p95) |
+|----------|-----------|-----------|--------------|---------------|
+| Quick    | 2         | 2         | 200          | <10ms         |
+| Standard | 5         | 3         | 2,500        | <20ms         |
+| Stress   | 20        | 10        | 40,000       | <50ms         |
+| Endurance| 10        | 5         | 10,000       | <30ms         |
+
+*Results vary based on hardware, network, and SeaweedFS configuration*
+
+### Tuning for Maximum Performance
+```yaml
+producers:
+  batch_size: 1000
+  linger_ms: 10
+  compression_type: "lz4"
+  acks: "1"  # Balance between speed and durability
+
+consumers:  
+  max_poll_records: 5000
+  fetch_min_bytes: 1048576  # 1MB
+  fetch_max_wait_ms: 100
+```
+
+## Comparison with Existing Tests
+
+| Feature | SMQ Tests | **Kafka Client Load Test** |
+|---------|-----------|----------------------------|
+| Protocol | SMQ (SeaweedFS native) | **Kafka (industry standard)** |
+| Clients | SMQ clients | **Real Kafka clients (Sarama, Confluent)** |
+| Schema Registry | ❌ | **✅ Full Avro/Protobuf support** |
+| Consumer Groups | Basic | **✅ Full Kafka consumer group features** |
+| Monitoring | Basic | **✅ Prometheus + Grafana dashboards** |
+| Test Scenarios | Limited | **✅ Multiple predefined scenarios** |
+| Real-world | Synthetic | **✅ Production-like workloads** |
+
+This load test provides comprehensive validation of the SeaweedFS Kafka Gateway using real-world Kafka clients and protocols.
+
+---
+
+## Quick Reference
+
+```bash
+# Essential Commands
+make help                    # Show all available commands
+make test                    # Run default comprehensive test  
+make quick-test              # 1-minute smoke test
+make stress-test             # High-load stress test
+make test-with-monitoring    # Include Grafana dashboards
+make clean                   # Clean up all resources
+
+# Monitoring
+make monitor                 # Start Prometheus + Grafana
+# → http://localhost:9090 (Prometheus)
+# → http://localhost:3000 (Grafana, admin/admin)
+
+# Advanced
+make benchmark               # Run full benchmark suite
+make health-check            # Validate service health
+make validate-setup          # Check configuration
+```

test/kafka/kafka-client-loadtest/SeekToBeginningTest.java

@@ -0,0 +1,179 @@
+import org.apache.kafka.clients.consumer.*;
+import org.apache.kafka.clients.consumer.internals.*;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.errors.TimeoutException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.*;
+
+/**
+ * Enhanced test program to reproduce and diagnose the seekToBeginning() hang issue
+ * 
+ * This test:
+ * 1. Adds detailed logging of Kafka client operations
+ * 2. Captures exceptions and timeouts
+ * 3. Shows what the consumer is waiting for
+ * 4. Tracks request/response lifecycle
+ */
+public class SeekToBeginningTest {
+    private static final Logger log = LoggerFactory.getLogger(SeekToBeginningTest.class);
+    
+    public static void main(String[] args) throws Exception {
+        String bootstrapServers = "localhost:9093";
+        String topicName = "_schemas";
+
+        if (args.length > 0) {
+            bootstrapServers = args[0];
+        }
+
+        Properties props = new Properties();
+        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+        props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-seek-group");
+        props.put(ConsumerConfig.CLIENT_ID_CONFIG, "test-seek-client");
+        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);
+        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);
+        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "45000");
+        props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, "60000");
+        
+        // Add comprehensive debug logging
+        props.put("log4j.logger.org.apache.kafka.clients.consumer.internals", "DEBUG");
+        props.put("log4j.logger.org.apache.kafka.clients.producer.internals", "DEBUG");
+        props.put("log4j.logger.org.apache.kafka.clients.Metadata", "DEBUG");
+        
+        // Add shorter timeouts to fail faster
+        props.put(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "10000"); // 10 seconds instead of 60
+        
+        System.out.println("\n╔════════════════════════════════════════════════════════════╗");
+        System.out.println("║         SeekToBeginning Diagnostic Test                      ║");
+        System.out.println(String.format("║     Connecting to: %-42s║", bootstrapServers));
+        System.out.println("╚════════════════════════════════════════════════════════════╝\n");
+
+        System.out.println("[TEST] Creating KafkaConsumer...");
+        System.out.println("[TEST] Bootstrap servers: " + bootstrapServers);
+        System.out.println("[TEST] Group ID: test-seek-group");
+        System.out.println("[TEST] Client ID: test-seek-client");
+        
+        KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(props);
+
+        TopicPartition tp = new TopicPartition(topicName, 0);
+        List<TopicPartition> partitions = Arrays.asList(tp);
+
+        System.out.println("\n[STEP 1] Assigning to partition: " + tp);
+        consumer.assign(partitions);
+        System.out.println("[STEP 1] ✓ Assigned successfully");
+
+        System.out.println("\n[STEP 2] Calling seekToBeginning()...");
+        long startTime = System.currentTimeMillis();
+        try {
+            consumer.seekToBeginning(partitions);
+            long seekTime = System.currentTimeMillis() - startTime;
+            System.out.println("[STEP 2] ✓ seekToBeginning() completed in " + seekTime + "ms");
+        } catch (Exception e) {
+            System.out.println("[STEP 2] ✗ EXCEPTION in seekToBeginning():");
+            e.printStackTrace();
+            consumer.close();
+            return;
+        }
+
+        System.out.println("\n[STEP 3] Starting poll loop...");
+        System.out.println("[STEP 3] First poll will trigger offset lookup (ListOffsets)");
+        System.out.println("[STEP 3] Then will fetch initial records\n");
+        
+        int successfulPolls = 0;
+        int failedPolls = 0;
+        int totalRecords = 0;
+
+        for (int i = 0; i < 3; i++) {
+            System.out.println("═══════════════════════════════════════════════════════════");
+            System.out.println("[POLL " + (i + 1) + "] Starting poll with 15-second timeout...");
+            long pollStart = System.currentTimeMillis();
+            
+            try {
+                System.out.println("[POLL " + (i + 1) + "] Calling consumer.poll()...");
+                ConsumerRecords<byte[], byte[]> records = consumer.poll(java.time.Duration.ofSeconds(15));
+                long pollTime = System.currentTimeMillis() - pollStart;
+                
+                System.out.println("[POLL " + (i + 1) + "] ✓ Poll completed in " + pollTime + "ms");
+                System.out.println("[POLL " + (i + 1) + "] Records received: " + records.count());
+                
+                if (records.count() > 0) {
+                    successfulPolls++;
+                    totalRecords += records.count();
+                    for (ConsumerRecord<byte[], byte[]> record : records) {
+                        System.out.println("  [RECORD] offset=" + record.offset() + 
+                                         ", key.len=" + (record.key() != null ? record.key().length : 0) +
+                                         ", value.len=" + (record.value() != null ? record.value().length : 0));
+                    }
+                } else {
+                    System.out.println("[POLL " + (i + 1) + "] ℹ No records in this poll (but no error)");
+                    successfulPolls++;
+                }
+            } catch (TimeoutException e) {
+                long pollTime = System.currentTimeMillis() - pollStart;
+                failedPolls++;
+                System.out.println("[POLL " + (i + 1) + "] ✗ TIMEOUT after " + pollTime + "ms");
+                System.out.println("[POLL " + (i + 1) + "] This means consumer is waiting for something from broker");
+                System.out.println("[POLL " + (i + 1) + "] Possible causes:");
+                System.out.println("         - ListOffsetsRequest never sent");
+                System.out.println("         - ListOffsetsResponse not received");
+                System.out.println("         - Broker metadata parsing failed");
+                System.out.println("         - Connection issue");
+                
+                // Print current position info if available
+                try {
+                    long position = consumer.position(tp);
+                    System.out.println("[POLL " + (i + 1) + "] Current position: " + position);
+                } catch (Exception e2) {
+                    System.out.println("[POLL " + (i + 1) + "] Could not get position: " + e2.getMessage());
+                }
+            } catch (Exception e) {
+                failedPolls++;
+                long pollTime = System.currentTimeMillis() - pollStart;
+                System.out.println("[POLL " + (i + 1) + "] ✗ EXCEPTION after " + pollTime + "ms:");
+                System.out.println("[POLL " + (i + 1) + "] Exception type: " + e.getClass().getSimpleName());
+                System.out.println("[POLL " + (i + 1) + "] Message: " + e.getMessage());
+                
+                // Print stack trace for first exception
+                if (i == 0) {
+                    System.out.println("[POLL " + (i + 1) + "] Stack trace:");
+                    e.printStackTrace();
+                }
+            }
+        }
+
+        System.out.println("\n═══════════════════════════════════════════════════════════");
+        System.out.println("[RESULTS] Test Summary:");
+        System.out.println("  Successful polls: " + successfulPolls);
+        System.out.println("  Failed polls: " + failedPolls);
+        System.out.println("  Total records received: " + totalRecords);
+        
+        if (failedPolls > 0) {
+            System.out.println("\n[DIAGNOSIS] Consumer is BLOCKED during poll()");
+            System.out.println("  This indicates the consumer cannot:");
+            System.out.println("  1. Send ListOffsetsRequest to determine offset 0, OR");
+            System.out.println("  2. Receive/parse ListOffsetsResponse from broker, OR");
+            System.out.println("  3. Parse broker metadata for partition leader lookup");
+        } else if (totalRecords == 0) {
+            System.out.println("\n[DIAGNOSIS] Consumer is working but NO records found");
+            System.out.println("  This might mean:");
+            System.out.println("  1. Topic has no messages, OR");
+            System.out.println("  2. Fetch is working but broker returns empty");
+        } else {
+            System.out.println("\n[SUCCESS] Consumer working correctly!");
+            System.out.println("  Received " + totalRecords + " records");
+        }
+
+        System.out.println("\n[CLEANUP] Closing consumer...");
+        try {
+            consumer.close();
+            System.out.println("[CLEANUP] ✓ Consumer closed successfully");
+        } catch (Exception e) {
+            System.out.println("[CLEANUP] ✗ Error closing consumer: " + e.getMessage());
+        }
+        
+        System.out.println("\n[TEST] Done!\n");
+    }
+}

test/kafka/kafka-client-loadtest/cmd/loadtest/main.go

@@ -0,0 +1,502 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/config"
+	"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/consumer"
+	"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/metrics"
+	"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/producer"
+	"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/schema"
+	"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/tracker"
+)
+
+var (
+	configFile = flag.String("config", "/config/loadtest.yaml", "Path to configuration file")
+	testMode   = flag.String("mode", "", "Test mode override (producer|consumer|comprehensive)")
+	duration   = flag.Duration("duration", 0, "Test duration override")
+	help       = flag.Bool("help", false, "Show help")
+)
+
+func main() {
+	flag.Parse()
+
+	if *help {
+		printHelp()
+		return
+	}
+
+	// Load configuration
+	cfg, err := config.Load(*configFile)
+	if err != nil {
+		log.Fatalf("Failed to load configuration: %v", err)
+	}
+
+	// Override configuration with environment variables and flags
+	cfg.ApplyOverrides(*testMode, *duration)
+
+	// Initialize metrics
+	metricsCollector := metrics.NewCollector()
+
+	// Start metrics HTTP server
+	go func() {
+		http.Handle("/metrics", promhttp.Handler())
+		http.HandleFunc("/health", healthCheck)
+		http.HandleFunc("/stats", func(w http.ResponseWriter, r *http.Request) {
+			metricsCollector.WriteStats(w)
+		})
+
+		log.Printf("Starting metrics server on :8080")
+		if err := http.ListenAndServe(":8080", nil); err != nil {
+			log.Printf("Metrics server error: %v", err)
+		}
+	}()
+
+	// Set up signal handling
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+
+	log.Printf("Starting Kafka Client Load Test")
+	log.Printf("Mode: %s, Duration: %v", cfg.TestMode, cfg.Duration)
+	log.Printf("Kafka Brokers: %v", cfg.Kafka.BootstrapServers)
+	log.Printf("Schema Registry: %s", cfg.SchemaRegistry.URL)
+	log.Printf("Schemas Enabled: %v", cfg.Schemas.Enabled)
+
+	// Register schemas if enabled
+	if cfg.Schemas.Enabled {
+		log.Printf("Registering schemas with Schema Registry...")
+		if err := registerSchemas(cfg); err != nil {
+			log.Fatalf("Failed to register schemas: %v", err)
+		}
+		log.Printf("Schemas registered successfully")
+	}
+
+	var wg sync.WaitGroup
+
+	// Start test based on mode
+	var testErr error
+	switch cfg.TestMode {
+	case "producer":
+		testErr = runProducerTest(ctx, cfg, metricsCollector, &wg)
+	case "consumer":
+		testErr = runConsumerTest(ctx, cfg, metricsCollector, &wg)
+	case "comprehensive":
+		testErr = runComprehensiveTest(ctx, cancel, cfg, metricsCollector, &wg)
+	default:
+		log.Fatalf("Unknown test mode: %s", cfg.TestMode)
+	}
+
+	// If test returned an error (e.g., circuit breaker), exit
+	if testErr != nil {
+		log.Printf("Test failed with error: %v", testErr)
+		cancel() // Cancel context to stop any remaining goroutines
+		return
+	}
+
+	// Wait for completion or signal
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-sigCh:
+		log.Printf("Received shutdown signal, stopping tests...")
+		cancel()
+
+		// Wait for graceful shutdown with timeout
+		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer shutdownCancel()
+
+		select {
+		case <-done:
+			log.Printf("All tests completed gracefully")
+		case <-shutdownCtx.Done():
+			log.Printf("Shutdown timeout, forcing exit")
+		}
+	case <-done:
+		log.Printf("All tests completed")
+	}
+
+	// Print final statistics
+	log.Printf("Final Test Statistics:")
+	metricsCollector.PrintSummary()
+}
+
+func runProducerTest(ctx context.Context, cfg *config.Config, collector *metrics.Collector, wg *sync.WaitGroup) error {
+	log.Printf("Starting producer-only test with %d producers", cfg.Producers.Count)
+
+	// Create record tracker with current timestamp to filter old messages
+	testStartTime := time.Now().UnixNano()
+	recordTracker := tracker.NewTracker("/test-results/produced.jsonl", "/test-results/consumed.jsonl", testStartTime)
+
+	errChan := make(chan error, cfg.Producers.Count)
+
+	for i := 0; i < cfg.Producers.Count; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+
+			prod, err := producer.New(cfg, collector, id, recordTracker)
+			if err != nil {
+				log.Printf("Failed to create producer %d: %v", id, err)
+				errChan <- err
+				return
+			}
+			defer prod.Close()
+
+			if err := prod.Run(ctx); err != nil {
+				log.Printf("Producer %d failed: %v", id, err)
+				errChan <- err
+				return
+			}
+		}(i)
+	}
+
+	// Wait for any producer error
+	select {
+	case err := <-errChan:
+		log.Printf("Producer test failed: %v", err)
+		return err
+	default:
+		return nil
+	}
+}
+
+func runConsumerTest(ctx context.Context, cfg *config.Config, collector *metrics.Collector, wg *sync.WaitGroup) error {
+	log.Printf("Starting consumer-only test with %d consumers", cfg.Consumers.Count)
+
+	// Create record tracker with current timestamp to filter old messages
+	testStartTime := time.Now().UnixNano()
+	recordTracker := tracker.NewTracker("/test-results/produced.jsonl", "/test-results/consumed.jsonl", testStartTime)
+
+	errChan := make(chan error, cfg.Consumers.Count)
+
+	for i := 0; i < cfg.Consumers.Count; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+
+			cons, err := consumer.New(cfg, collector, id, recordTracker)
+			if err != nil {
+				log.Printf("Failed to create consumer %d: %v", id, err)
+				errChan <- err
+				return
+			}
+			defer cons.Close()
+
+			cons.Run(ctx)
+		}(i)
+	}
+
+	// Consumers don't typically return errors in the same way, so just return nil
+	return nil
+}
+
+func runComprehensiveTest(ctx context.Context, cancel context.CancelFunc, cfg *config.Config, collector *metrics.Collector, wg *sync.WaitGroup) error {
+	log.Printf("Starting comprehensive test with %d producers and %d consumers",
+		cfg.Producers.Count, cfg.Consumers.Count)
+
+	// Create record tracker with current timestamp to filter old messages
+	testStartTime := time.Now().UnixNano()
+	log.Printf("Test run starting at %d - only tracking messages from this run", testStartTime)
+	recordTracker := tracker.NewTracker("/test-results/produced.jsonl", "/test-results/consumed.jsonl", testStartTime)
+
+	errChan := make(chan error, cfg.Producers.Count)
+
+	// Create separate contexts for producers and consumers
+	producerCtx, producerCancel := context.WithCancel(ctx)
+	consumerCtx, consumerCancel := context.WithCancel(ctx)
+
+	// Start producers
+	for i := 0; i < cfg.Producers.Count; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+
+			prod, err := producer.New(cfg, collector, id, recordTracker)
+			if err != nil {
+				log.Printf("Failed to create producer %d: %v", id, err)
+				errChan <- err
+				return
+			}
+			defer prod.Close()
+
+			if err := prod.Run(producerCtx); err != nil {
+				log.Printf("Producer %d failed: %v", id, err)
+				errChan <- err
+				return
+			}
+		}(i)
+	}
+
+	// Wait briefly for producers to start producing messages
+	// Reduced from 5s to 2s to minimize message backlog
+	time.Sleep(2 * time.Second)
+
+	// Start consumers
+	// NOTE: With unique ClientIDs, all consumers can start simultaneously without connection storms
+	for i := 0; i < cfg.Consumers.Count; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+
+			cons, err := consumer.New(cfg, collector, id, recordTracker)
+			if err != nil {
+				log.Printf("Failed to create consumer %d: %v", id, err)
+				return
+			}
+			defer cons.Close()
+
+			cons.Run(consumerCtx)
+		}(i)
+	}
+
+	// Check for producer errors
+	select {
+	case err := <-errChan:
+		log.Printf("Comprehensive test failed due to producer error: %v", err)
+		producerCancel()
+		consumerCancel()
+		return err
+	default:
+		// No immediate error, continue
+	}
+
+	// If duration is set, stop producers first, then allow consumers extra time to drain
+	if cfg.Duration > 0 {
+		go func() {
+			timer := time.NewTimer(cfg.Duration)
+			defer timer.Stop()
+
+			select {
+			case <-timer.C:
+				log.Printf("Test duration (%v) reached, stopping producers", cfg.Duration)
+				producerCancel()
+
+				// Allow consumers extra time to drain remaining messages
+				// Calculate drain time based on test duration (minimum 60s, up to test duration)
+				drainTime := 60 * time.Second
+				if cfg.Duration > drainTime {
+					drainTime = cfg.Duration // Match test duration for longer tests
+				}
+				log.Printf("Allowing %v for consumers to drain remaining messages...", drainTime)
+				time.Sleep(drainTime)
+
+				log.Printf("Stopping consumers after drain period")
+				consumerCancel()
+				cancel()
+			case <-ctx.Done():
+				// Context already cancelled
+				producerCancel()
+				consumerCancel()
+			}
+		}()
+	} else {
+		// No duration set, wait for cancellation and ensure cleanup
+		go func() {
+			<-ctx.Done()
+			producerCancel()
+			consumerCancel()
+		}()
+	}
+
+	// Wait for all producer and consumer goroutines to complete
+	log.Printf("Waiting for all producers and consumers to complete...")
+	wg.Wait()
+	log.Printf("All producers and consumers completed, starting verification...")
+
+	// Save produced and consumed records
+	log.Printf("Saving produced records...")
+	if err := recordTracker.SaveProduced(); err != nil {
+		log.Printf("Failed to save produced records: %v", err)
+	}
+
+	log.Printf("Saving consumed records...")
+	if err := recordTracker.SaveConsumed(); err != nil {
+		log.Printf("Failed to save consumed records: %v", err)
+	}
+
+	// Compare records
+	log.Printf("Comparing produced vs consumed records...")
+	result := recordTracker.Compare()
+	result.PrintSummary()
+
+	log.Printf("Verification complete!")
+	return nil
+}
+
+func healthCheck(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, "OK")
+}
+
+func printHelp() {
+	fmt.Printf(`Kafka Client Load Test for SeaweedFS
+
+Usage: %s [options]
+
+Options:
+  -config string
+        Path to configuration file (default "/config/loadtest.yaml")
+  -mode string
+        Test mode override (producer|consumer|comprehensive)
+  -duration duration
+        Test duration override
+  -help
+        Show this help message
+
+Environment Variables:
+  KAFKA_BOOTSTRAP_SERVERS  Comma-separated list of Kafka brokers
+  SCHEMA_REGISTRY_URL      URL of the Schema Registry
+  TEST_DURATION           Test duration (e.g., "5m", "300s")
+  TEST_MODE               Test mode (producer|consumer|comprehensive)
+  PRODUCER_COUNT          Number of producer instances
+  CONSUMER_COUNT          Number of consumer instances
+  MESSAGE_RATE            Messages per second per producer
+  MESSAGE_SIZE            Message size in bytes
+  TOPIC_COUNT             Number of topics to create
+  PARTITIONS_PER_TOPIC    Number of partitions per topic
+  VALUE_TYPE              Message value type (json/avro/binary)
+
+Test Modes:
+  producer       - Run only producers (generate load)
+  consumer       - Run only consumers (consume existing messages)
+  comprehensive  - Run both producers and consumers simultaneously
+
+Example:
+  %s -config ./config/loadtest.yaml -mode comprehensive -duration 10m
+
+`, os.Args[0], os.Args[0])
+}
+
+// registerSchemas registers schemas with Schema Registry for all topics
+func registerSchemas(cfg *config.Config) error {
+	// Wait for Schema Registry to be ready
+	if err := waitForSchemaRegistry(cfg.SchemaRegistry.URL); err != nil {
+		return fmt.Errorf("schema registry not ready: %w", err)
+	}
+
+	// Register schemas for each topic with different formats for variety
+	topics := cfg.GetTopicNames()
+
+	// Determine schema formats - use different formats for different topics
+	// This provides comprehensive testing of all schema format variations
+	for i, topic := range topics {
+		var schemaFormat string
+
+		// Distribute topics across three schema formats for comprehensive testing
+		// Format 0: AVRO (default, most common)
+		// Format 1: JSON (modern, human-readable)
+		// Format 2: PROTOBUF (efficient binary format)
+		switch i % 3 {
+		case 0:
+			schemaFormat = "AVRO"
+		case 1:
+			schemaFormat = "JSON"
+		case 2:
+			schemaFormat = "PROTOBUF"
+		}
+
+		// Allow override from config if specified
+		if cfg.Producers.SchemaFormat != "" {
+			schemaFormat = cfg.Producers.SchemaFormat
+		}
+
+		if err := registerTopicSchema(cfg.SchemaRegistry.URL, topic, schemaFormat); err != nil {
+			return fmt.Errorf("failed to register schema for topic %s (format: %s): %w", topic, schemaFormat, err)
+		}
+		log.Printf("Schema registered for topic %s with format: %s", topic, schemaFormat)
+	}
+
+	return nil
+}
+
+// waitForSchemaRegistry waits for Schema Registry to be ready
+func waitForSchemaRegistry(url string) error {
+	maxRetries := 30
+	for i := 0; i < maxRetries; i++ {
+		resp, err := http.Get(url + "/subjects")
+		if err == nil && resp.StatusCode == 200 {
+			resp.Body.Close()
+			return nil
+		}
+		if resp != nil {
+			resp.Body.Close()
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return fmt.Errorf("schema registry not ready after %d retries", maxRetries)
+}
+
+// registerTopicSchema registers a schema for a specific topic
+func registerTopicSchema(registryURL, topicName, schemaFormat string) error {
+	// Determine schema format, default to AVRO
+	if schemaFormat == "" {
+		schemaFormat = "AVRO"
+	}
+
+	var schemaStr string
+	var schemaType string
+
+	switch strings.ToUpper(schemaFormat) {
+	case "AVRO":
+		schemaStr = schema.GetAvroSchema()
+		schemaType = "AVRO"
+	case "JSON", "JSON_SCHEMA":
+		schemaStr = schema.GetJSONSchema()
+		schemaType = "JSON"
+	case "PROTOBUF":
+		schemaStr = schema.GetProtobufSchema()
+		schemaType = "PROTOBUF"
+	default:
+		return fmt.Errorf("unsupported schema format: %s", schemaFormat)
+	}
+
+	schemaReq := map[string]interface{}{
+		"schema":     schemaStr,
+		"schemaType": schemaType,
+	}
+
+	jsonData, err := json.Marshal(schemaReq)
+	if err != nil {
+		return err
+	}
+
+	// Register schema for topic value
+	subject := topicName + "-value"
+	url := fmt.Sprintf("%s/subjects/%s/versions", registryURL, subject)
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Post(url, "application/vnd.schemaregistry.v1+json", bytes.NewBuffer(jsonData))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("schema registration failed: status=%d, body=%s", resp.StatusCode, string(body))
+	}
+
+	log.Printf("Schema registered for topic %s (format: %s)", topicName, schemaType)
+	return nil
+}

test/kafka/kafka-client-loadtest/config/loadtest.yaml

@@ -0,0 +1,169 @@
+# Kafka Client Load Test Configuration
+
+# Test execution settings
+test_mode: "comprehensive"  # producer, consumer, comprehensive
+duration: "60s"  # Test duration (0 = run indefinitely) - producers will stop at this time, consumers get +120s to drain
+
+# Kafka cluster configuration
+kafka:
+  bootstrap_servers:
+    - "kafka-gateway:9093"
+  # Security settings (if needed)
+  security_protocol: "PLAINTEXT"  # PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL
+  sasl_mechanism: ""  # PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
+  sasl_username: ""
+  sasl_password: ""
+
+# Schema Registry configuration
+schema_registry:
+  url: "http://schema-registry:8081"
+  auth:
+    username: ""
+    password: ""
+
+# Producer configuration
+producers:
+  count: 10  # Number of producer instances
+  message_rate: 1000  # Messages per second per producer
+  message_size: 1024  # Message size in bytes
+  batch_size: 100  # Batch size for batching
+  linger_ms: 5  # Time to wait for batching
+  compression_type: "snappy"  # none, gzip, snappy, lz4, zstd
+  acks: "all"  # 0, 1, all
+  retries: 3
+  retry_backoff_ms: 100
+  request_timeout_ms: 30000
+  delivery_timeout_ms: 120000
+  
+  # Message generation settings
+  key_distribution: "random"  # random, sequential, uuid
+  value_type: "avro"  # json, avro, protobuf, binary
+  schema_format: ""  # AVRO, JSON, PROTOBUF - schema registry format (when schemas enabled)
+                     # Leave empty to auto-distribute formats across topics for testing:
+                     #   topic-0: AVRO, topic-1: JSON, topic-2: PROTOBUF, topic-3: AVRO, topic-4: JSON
+                     # Set to specific format (e.g. "AVRO") to use same format for all topics
+  include_timestamp: true
+  include_headers: true
+
+# Consumer configuration  
+consumers:
+  count: 5  # Number of consumer instances
+  group_prefix: "loadtest-group"  # Consumer group prefix
+  auto_offset_reset: "earliest"  # earliest, latest
+  enable_auto_commit: true
+  auto_commit_interval_ms: 100  # Reduced from 1000ms to 100ms to minimize duplicate window
+  session_timeout_ms: 30000
+  heartbeat_interval_ms: 3000
+  max_poll_records: 500
+  max_poll_interval_ms: 300000
+  fetch_min_bytes: 1
+  fetch_max_bytes: 52428800  # 50MB
+  fetch_max_wait_ms: 100  # 100ms - very fast polling for concurrent fetches and quick drain
+
+# Topic configuration
+topics:
+  count: 5  # Number of topics to create/use
+  prefix: "loadtest-topic"  # Topic name prefix
+  partitions: 4  # Partitions per topic (default: 4)
+  replication_factor: 1  # Replication factor
+  cleanup_policy: "delete"  # delete, compact
+  retention_ms: 604800000  # 7 days
+  segment_ms: 86400000  # 1 day
+
+# Schema configuration (for Avro/Protobuf tests)
+schemas:
+  enabled: true
+  registry_timeout_ms: 10000
+  
+  # Test schemas
+  user_event:
+    type: "avro"
+    schema: |
+      {
+        "type": "record",
+        "name": "UserEvent",
+        "namespace": "com.seaweedfs.test",
+        "fields": [
+          {"name": "user_id", "type": "string"},
+          {"name": "event_type", "type": "string"},
+          {"name": "timestamp", "type": "long"},
+          {"name": "properties", "type": {"type": "map", "values": "string"}}
+        ]
+      }
+      
+  transaction:
+    type: "avro" 
+    schema: |
+      {
+        "type": "record",
+        "name": "Transaction", 
+        "namespace": "com.seaweedfs.test",
+        "fields": [
+          {"name": "transaction_id", "type": "string"},
+          {"name": "amount", "type": "double"},
+          {"name": "currency", "type": "string"},
+          {"name": "merchant_id", "type": "string"},
+          {"name": "timestamp", "type": "long"}
+        ]
+      }
+
+# Metrics and monitoring
+metrics:
+  enabled: true
+  collection_interval: "10s"
+  prometheus_port: 8080
+  
+  # What to measure
+  track_latency: true
+  track_throughput: true
+  track_errors: true
+  track_consumer_lag: true
+  
+  # Latency percentiles to track
+  latency_percentiles: [50, 90, 95, 99, 99.9]
+
+# Load test scenarios
+scenarios:
+  # Steady state load test
+  steady_load:
+    producer_rate: 1000  # messages/sec per producer
+    ramp_up_time: "30s"
+    steady_duration: "240s" 
+    ramp_down_time: "30s"
+    
+  # Burst load test  
+  burst_load:
+    base_rate: 500
+    burst_rate: 5000
+    burst_duration: "10s"
+    burst_interval: "60s"
+    
+  # Gradual ramp test
+  ramp_test:
+    start_rate: 100
+    end_rate: 2000
+    ramp_duration: "300s"
+    step_duration: "30s"
+
+# Error injection (for resilience testing)
+chaos:
+  enabled: false
+  producer_failure_rate: 0.01  # 1% of producers fail randomly
+  consumer_failure_rate: 0.01  # 1% of consumers fail randomly
+  network_partition_probability: 0.001  # Network issues
+  broker_restart_interval: "0s"  # Restart brokers periodically (0s = disabled)
+
+# Output and reporting
+output:
+  results_dir: "/test-results"
+  export_prometheus: true
+  export_csv: true
+  export_json: true
+  real_time_stats: true
+  stats_interval: "30s"
+  
+# Logging
+logging:
+  level: "info"  # debug, info, warn, error
+  format: "text"  # text, json
+  enable_kafka_logs: false  # Enable Kafka client debug logs

test/kafka/kafka-client-loadtest/docker-compose-kafka-compare.yml

@@ -0,0 +1,46 @@
+version: '3.8'
+
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.5.0
+    hostname: zookeeper
+    container_name: compare-zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  kafka:
+    image: confluentinc/cp-kafka:7.5.0
+    hostname: kafka
+    container_name: compare-kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_LOG_RETENTION_HOURS: 1
+      KAFKA_LOG_SEGMENT_BYTES: 1073741824
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry:7.5.0
+    hostname: schema-registry
+    container_name: compare-schema-registry
+    depends_on:
+      - kafka
+    ports:
+      - "8082:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:29092'
+      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+

test/kafka/kafka-client-loadtest/docker-compose.yml

@@ -0,0 +1,336 @@
+# SeaweedFS Kafka Client Load Test
+# Tests the full stack: Kafka Clients -> SeaweedFS Kafka Gateway -> SeaweedFS MQ Broker -> Storage
+
+x-seaweedfs-build: &seaweedfs-build
+  build:
+    context: .
+    dockerfile: Dockerfile.seaweedfs
+    args:
+      TARGETARCH: ${GOARCH:-arm64}
+      CACHE_BUST: ${CACHE_BUST:-latest}
+  image: kafka-client-loadtest-seaweedfs
+
+services:
+  # Schema Registry (for Avro/Protobuf support) 
+  # Using host networking to connect to localhost:9093 (where our gateway advertises)
+  # WORKAROUND: Schema Registry hangs on empty _schemas topic during bootstrap
+  # Pre-create the topic first to avoid "wait to catch up" hang
+  schema-registry-init:
+    image: confluentinc/cp-kafka:8.0.0
+    container_name: loadtest-schema-registry-init
+    networks:
+      - kafka-loadtest-net
+    depends_on:
+      kafka-gateway:
+        condition: service_healthy
+    command: >
+      bash -c "
+      echo 'Creating _schemas topic...';
+      kafka-topics --create --topic _schemas --partitions 1 --replication-factor 1 --bootstrap-server kafka-gateway:9093 --if-not-exists || exit 0;
+      echo '_schemas topic created successfully';
+      "
+  
+  schema-registry:
+    image: confluentinc/cp-schema-registry:8.0.0
+    container_name: loadtest-schema-registry
+    restart: on-failure:3
+    ports:
+      - "8081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_HOST_PORT: 8081
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka-gateway:9093'
+      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
+      SCHEMA_REGISTRY_DEBUG: "true"
+      SCHEMA_REGISTRY_SCHEMA_COMPATIBILITY_LEVEL: "full"
+      SCHEMA_REGISTRY_LEADER_ELIGIBILITY: "true"
+      SCHEMA_REGISTRY_MODE: "READWRITE"
+      SCHEMA_REGISTRY_GROUP_ID: "schema-registry"
+      SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID: "schema-registry"
+      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: "PLAINTEXT"
+      SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: "1"
+      SCHEMA_REGISTRY_KAFKASTORE_INIT_TIMEOUT: "120000"
+      SCHEMA_REGISTRY_KAFKASTORE_TIMEOUT: "60000"
+      SCHEMA_REGISTRY_REQUEST_TIMEOUT_MS: "60000"
+      SCHEMA_REGISTRY_RETRY_BACKOFF_MS: "1000"
+      # Force IPv4 to work around Java IPv6 issues
+      # Enable verbose logging and set reasonable memory limits
+      KAFKA_OPTS: "-Djava.net.preferIPv4Stack=true -Djava.net.preferIPv4Addresses=true -Xmx512M -Xms256M"
+      KAFKA_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/kafka/log4j.properties"
+      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: "INFO"
+      SCHEMA_REGISTRY_KAFKASTORE_WRITE_TIMEOUT_MS: "60000"
+      SCHEMA_REGISTRY_KAFKASTORE_INIT_RETRY_BACKOFF_MS: "5000"
+      SCHEMA_REGISTRY_KAFKASTORE_CONSUMER_AUTO_OFFSET_RESET: "earliest"
+      # Enable comprehensive Kafka client DEBUG logging to trace offset management
+      SCHEMA_REGISTRY_LOG4J_LOGGERS: "org.apache.kafka.clients.consumer.internals.OffsetsRequestManager=DEBUG,org.apache.kafka.clients.consumer.internals.Fetcher=DEBUG,org.apache.kafka.clients.consumer.internals.AbstractFetch=DEBUG,org.apache.kafka.clients.Metadata=DEBUG,org.apache.kafka.common.network=DEBUG"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
+      interval: 15s
+      timeout: 10s
+      retries: 10
+      start_period: 30s
+    depends_on:
+      schema-registry-init:
+        condition: service_completed_successfully
+      kafka-gateway:
+        condition: service_healthy
+    networks:
+      - kafka-loadtest-net
+
+  # SeaweedFS Master (coordinator)
+  seaweedfs-master:
+    <<: *seaweedfs-build
+    container_name: loadtest-seaweedfs-master
+    ports:
+      - "9333:9333"
+      - "19333:19333"
+    command: 
+      - master
+      - -ip=seaweedfs-master
+      - -port=9333
+      - -port.grpc=19333
+      - -volumeSizeLimitMB=48
+      - -defaultReplication=000
+      - -garbageThreshold=0.3
+    volumes:
+      - ./data/seaweedfs-master:/data
+    healthcheck:
+      test: ["CMD-SHELL", "wget --quiet --tries=1 --spider http://seaweedfs-master:9333/cluster/status || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 20s
+    networks:
+      - kafka-loadtest-net
+
+  # SeaweedFS Volume Server (storage)
+  seaweedfs-volume:
+    <<: *seaweedfs-build
+    container_name: loadtest-seaweedfs-volume
+    ports:
+      - "8080:8080"
+      - "18080:18080"
+    command:
+      - volume
+      - -mserver=seaweedfs-master:9333
+      - -ip=seaweedfs-volume
+      - -port=8080
+      - -port.grpc=18080
+      - -publicUrl=seaweedfs-volume:8080
+      - -preStopSeconds=1
+      - -compactionMBps=50
+      - -max=0
+      - -dir=/data
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+    volumes:
+      - ./data/seaweedfs-volume:/data
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://seaweedfs-volume:8080/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 15s
+    networks:
+      - kafka-loadtest-net
+
+  # SeaweedFS Filer (metadata)
+  seaweedfs-filer:
+    <<: *seaweedfs-build
+    container_name: loadtest-seaweedfs-filer
+    ports:
+      - "8888:8888"
+      - "18888:18888"
+      - "18889:18889"
+    command:
+      - filer
+      - -master=seaweedfs-master:9333
+      - -ip=seaweedfs-filer
+      - -port=8888
+      - -port.grpc=18888
+      - -metricsPort=18889
+      - -defaultReplicaPlacement=000
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+      seaweedfs-volume:
+        condition: service_healthy
+    volumes:
+      - ./data/seaweedfs-filer:/data
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://seaweedfs-filer:8888/"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 15s
+    networks:
+      - kafka-loadtest-net
+
+  # SeaweedFS MQ Broker (message handling)
+  seaweedfs-mq-broker:
+    <<: *seaweedfs-build
+    container_name: loadtest-seaweedfs-mq-broker
+    ports:
+      - "17777:17777"
+      - "18777:18777"  # pprof profiling port
+    command:
+      - mq.broker
+      - -master=seaweedfs-master:9333
+      - -ip=seaweedfs-mq-broker
+      - -port=17777
+      - -logFlushInterval=0
+      - -port.pprof=18777
+    depends_on:
+      seaweedfs-filer:
+        condition: service_healthy
+    volumes:
+      - ./data/seaweedfs-mq:/data
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "17777"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+    networks:
+      - kafka-loadtest-net
+
+  # SeaweedFS Kafka Gateway (Kafka protocol compatibility)
+  kafka-gateway:
+    <<: *seaweedfs-build
+    container_name: loadtest-kafka-gateway
+    ports:
+      - "9093:9093"
+      - "10093:10093"  # pprof profiling port
+    command:
+      - mq.kafka.gateway
+      - -master=seaweedfs-master:9333
+      - -ip=kafka-gateway
+      - -ip.bind=0.0.0.0
+      - -port=9093
+      - -default-partitions=4
+      - -schema-registry-url=http://schema-registry:8081
+      - -port.pprof=10093
+    depends_on:
+      seaweedfs-filer:
+        condition: service_healthy
+      seaweedfs-mq-broker:
+        condition: service_healthy
+    environment:
+      - SEAWEEDFS_MASTERS=seaweedfs-master:9333
+      # - KAFKA_DEBUG=1  # Enable debug logging for Schema Registry troubleshooting
+      - KAFKA_ADVERTISED_HOST=kafka-gateway
+    volumes:
+      - ./data/kafka-gateway:/data
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "9093"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 45s  # Increased to account for 10s startup delay + filer discovery
+    networks:
+      - kafka-loadtest-net
+
+  # Kafka Client Load Test Runner
+  kafka-client-loadtest:
+    build:
+      context: ../../..
+      dockerfile: test/kafka/kafka-client-loadtest/Dockerfile.loadtest
+    container_name: kafka-client-loadtest-runner
+    depends_on:
+      kafka-gateway:
+        condition: service_healthy
+      # schema-registry:
+      #   condition: service_healthy
+    environment:
+      - KAFKA_BOOTSTRAP_SERVERS=kafka-gateway:9093
+      - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+      - TEST_DURATION=${TEST_DURATION:-300s}
+      - PRODUCER_COUNT=${PRODUCER_COUNT:-10}
+      - CONSUMER_COUNT=${CONSUMER_COUNT:-5}
+      - MESSAGE_RATE=${MESSAGE_RATE:-1000}
+      - MESSAGE_SIZE=${MESSAGE_SIZE:-1024}
+      - TOPIC_COUNT=${TOPIC_COUNT:-5}
+      - PARTITIONS_PER_TOPIC=${PARTITIONS_PER_TOPIC:-3}
+      - TEST_MODE=${TEST_MODE:-comprehensive}
+      - SCHEMAS_ENABLED=${SCHEMAS_ENABLED:-true}
+      - VALUE_TYPE=${VALUE_TYPE:-avro}
+    profiles:
+      - loadtest
+    volumes:
+      - ./test-results:/test-results
+    networks:
+      - kafka-loadtest-net
+
+  # Monitoring and Metrics
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: loadtest-prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus-data:/prometheus
+    networks:
+      - kafka-loadtest-net
+    profiles:
+      - monitoring
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: loadtest-grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning
+      - grafana-data:/var/lib/grafana
+    networks:
+      - kafka-loadtest-net
+    profiles:
+      - monitoring
+
+  # Schema Registry Debug Runner
+  schema-registry-debug:
+    build:
+      context: debug-client
+      dockerfile: Dockerfile
+    container_name: schema-registry-debug-runner
+    depends_on:
+      kafka-gateway:
+        condition: service_healthy
+    networks:
+      - kafka-loadtest-net
+    profiles:
+      - debug
+
+  # SeekToBeginning test - reproduces the hang issue
+  seek-test:
+    build:
+      context: .
+      dockerfile: Dockerfile.seektest
+    container_name: loadtest-seek-test
+    depends_on:
+      kafka-gateway:
+        condition: service_healthy
+      schema-registry:
+        condition: service_healthy
+    environment:
+      - KAFKA_BOOTSTRAP_SERVERS=kafka-gateway:9093
+    networks:
+      - kafka-loadtest-net
+    entrypoint: ["java", "-cp", "target/seek-test.jar", "SeekToBeginningTest"]
+    command: ["kafka-gateway:9093"]
+
+volumes:
+  prometheus-data:
+  grafana-data:
+
+networks:
+  kafka-loadtest-net:
+    driver: bridge
+    name: kafka-client-loadtest
+

test/kafka/kafka-client-loadtest/go.mod

@@ -0,0 +1,41 @@
+module github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest
+
+go 1.24.0
+
+toolchain go1.24.7
+
+require (
+	github.com/IBM/sarama v1.46.1
+	github.com/linkedin/goavro/v2 v2.14.0
+	github.com/prometheus/client_golang v1.23.2
+	google.golang.org/protobuf v1.36.8
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/eapache/go-resiliency v1.7.0 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/golang/snappy v1.0.0 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
+	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+	github.com/jcmturner/gofork v1.7.6 // indirect
+	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
+	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/crypto v0.43.0 // indirect
+	golang.org/x/net v0.46.0 // indirect
+	golang.org/x/sys v0.37.0 // indirect
+)