style: align const formatting in sra_transport.go

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.github/workflows/helm_ci.yml

@@ -27,7 +27,7 @@ jobs:
 
       - uses: actions/setup-python@v6
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           check-latest: true
 
       - name: Set up chart-testing

.github/workflows/s3-tables-tests.yml

@@ -122,6 +122,69 @@ jobs:
           path: test/s3tables/catalog/test-output.log
           retention-days: 3
 
+  trino-iceberg-catalog-tests:
+    name: Trino Iceberg Catalog Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+
+      - name: Pre-pull Trino image
+        run: docker pull trinodb/trino:479
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false ./weed
+
+      - name: Run Trino Iceberg Catalog Integration Tests
+        timeout-minutes: 25
+        working-directory: test/s3tables/catalog_trino
+        run: |
+          set -x
+          set -o pipefail
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting Trino Iceberg Catalog Tests ==="
+          
+          # Run Trino + Iceberg catalog integration tests
+          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
+            echo "Trino Iceberg catalog integration tests failed"
+            exit 1
+          }
+
+      - name: Show test output on failure
+        if: failure()
+        working-directory: test/s3tables/catalog_trino
+        run: |
+          echo "=== Test Output ==="
+          if [ -f test-output.log ]; then
+            tail -200 test-output.log
+          fi
+          
+          echo "=== Process information ==="
+          ps aux | grep -E "(weed|test|docker)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: trino-iceberg-catalog-test-logs
+          path: test/s3tables/catalog_trino/test-output.log
+          retention-days: 3
+
   s3-tables-build-verification:
     name: S3 Tables Build Verification
     runs-on: ubuntu-22.04

k8s/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml

@@ -0,0 +1,61 @@
+{{- define "seaweedfs.s3.iceberg.ingress.paths" -}}
+paths:
+- path: {{ .Values.s3.icebergIngress.path | quote }}
+  pathType: {{ .Values.s3.icebergIngress.pathType | quote }}
+  backend:
+    {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+    service:
+      name: {{ include "seaweedfs.componentName" (list . "s3") }}
+      port:
+        number: {{ .Values.s3.icebergPort }}
+    {{- else }}
+    serviceName: {{ include "seaweedfs.componentName" (list . "s3") }}
+    servicePort: {{ .Values.s3.icebergPort }}
+    {{- end }}
+{{- end -}}
+{{- if and .Values.s3.enabled .Values.s3.icebergPort .Values.s3.icebergIngress.enabled }}
+{{- $hosts := list }}
+{{- if kindIs "slice" .Values.s3.icebergIngress.host }}
+  {{- $hosts = .Values.s3.icebergIngress.host }}
+{{- else if .Values.s3.icebergIngress.host }}
+  {{- $hosts = list .Values.s3.icebergIngress.host }}
+{{- end }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: ingress-{{ include "seaweedfs.fullname" . }}-s3-iceberg
+  namespace: {{ .Release.Namespace }}
+  {{- with .Values.s3.icebergIngress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: s3-iceberg
+spec:
+  {{- if .Values.s3.icebergIngress.className }}
+  ingressClassName: {{ .Values.s3.icebergIngress.className | quote }}
+  {{- end }}
+  tls:
+    {{ .Values.s3.icebergIngress.tls | default list | toYaml | nindent 6}}
+  rules:
+{{- if $hosts }}
+{{- range $host := $hosts }}
+  - host: {{ $host | quote }}
+    http:
+      {{- include "seaweedfs.s3.iceberg.ingress.paths" $ | nindent 6 }}
+{{- end }}
+{{- else }}
+  - http:
+      {{- include "seaweedfs.s3.iceberg.ingress.paths" . | nindent 4 }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3/s3-service.yaml

@@ -16,6 +16,9 @@ metadata:
 {{- end }}
 spec:
   internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
+  {{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) (or .Values.s3.trafficDistribution .Values.filer.s3.trafficDistribution) }}
+  trafficDistribution: {{ include "seaweedfs.trafficDistribution" . }}
+  {{- end }}
   ports:
   - name: "swfs-s3"
     port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}

k8s/charts/seaweedfs/templates/shared/_helpers.tpl

@@ -323,3 +323,12 @@ Create the name of the service account to use
 {{- define "seaweedfs.serviceAccountName" -}}
 {{- .Values.global.serviceAccountName | default "seaweedfs" -}}
 {{- end -}}
+
+{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35 */}}
+{{- define "seaweedfs.trafficDistribution" -}}
+{{- if .Values.s3.trafficDistribution -}}
+{{- and (eq .Values.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.s3.trafficDistribution -}}
+{{- else if .Values.filer.s3.trafficDistribution -}}
+{{- and (eq .Values.filer.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.filer.s3.trafficDistribution -}}
+{{- end -}}
+{{- end -}}

k8s/charts/seaweedfs/values.yaml

@@ -1053,6 +1053,15 @@ s3:
     annotations: {}
     tls: []
 
+  icebergIngress:
+    enabled: false
+    className: ""
+    host: "seaweedfs-iceberg.cluster.local"
+    path: "/"
+    pathType: Prefix
+    annotations: {}
+    tls: []
+
 sftp:
   enabled: false
   imageOverride: null

test/s3/catalog_trino/trino_catalog_test.go

@@ -0,0 +1,413 @@
+package catalog_trino
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+type TestEnvironment struct {
+	seaweedDir      string
+	weedBinary      string
+	dataDir         string
+	bindIP          string
+	s3Port          int
+	s3GrpcPort      int
+	icebergPort     int
+	masterPort      int
+	masterGrpcPort  int
+	filerPort       int
+	filerGrpcPort   int
+	volumePort      int
+	volumeGrpcPort  int
+	weedProcess     *exec.Cmd
+	weedCancel      context.CancelFunc
+	trinoContainer  string
+	dockerAvailable bool
+}
+
+func TestTrinoIcebergCatalog(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Trino integration test")
+	}
+
+	env.StartSeaweedFS(t)
+
+	catalogBucket := "default"
+	createTableBucket(t, env, catalogBucket)
+	createObjectBucket(t, env, catalogBucket)
+
+	configDir := env.writeTrinoConfig(t, catalogBucket)
+	env.startTrinoContainer(t, configDir)
+	waitForTrino(t, env.trinoContainer, 60*time.Second)
+
+	schemaName := "trino_" + randomString(6)
+
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, schemaName) {
+		t.Fatalf("Expected schema %s in output:\n%s", schemaName, output)
+	}
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SHOW TABLES FROM iceberg.%s", schemaName))
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	if _, err := os.Stat(weedBinary); os.IsNotExist(err) {
+		weedBinary = "weed"
+		if _, err := exec.LookPath(weedBinary); err != nil {
+			t.Skip("weed binary not found, skipping integration test")
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-trino-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+
+	bindIP := findBindIP()
+
+	masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := mustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := mustFreePortPair(t, "S3")
+	icebergPort := mustFreePort(t, "Iceberg")
+
+	return &TestEnvironment{
+		seaweedDir:      seaweedDir,
+		weedBinary:      weedBinary,
+		dataDir:         dataDir,
+		bindIP:          bindIP,
+		s3Port:          s3Port,
+		s3GrpcPort:      s3GrpcPort,
+		icebergPort:     icebergPort,
+		masterPort:      masterPort,
+		masterGrpcPort:  masterGrpcPort,
+		filerPort:       filerPort,
+		filerGrpcPort:   filerGrpcPort,
+		volumePort:      volumePort,
+		volumeGrpcPort:  volumeGrpcPort,
+		dockerAvailable: hasDocker(),
+	}
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	securityToml := filepath.Join(env.dataDir, "security.toml")
+	if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
+		t.Fatalf("Failed to create security.toml: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
+		"-s3.iam.readOnly=false",
+		"-ip", env.bindIP,
+		"-ip.bind", env.bindIP,
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
+	}
+	env.weedProcess = cmd
+
+	if !env.waitForService(fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort), 30*time.Second) {
+		t.Fatalf("Iceberg REST API did not become ready")
+	}
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+
+	if env.trinoContainer != "" {
+		_ = exec.Command("docker", "rm", "-f", env.trinoContainer).Run()
+	}
+
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+
+	if env.weedProcess != nil {
+		time.Sleep(2 * time.Second)
+		_ = env.weedProcess.Wait()
+	}
+
+	if env.dataDir != "" {
+		_ = os.RemoveAll(env.dataDir)
+	}
+}
+
+func (env *TestEnvironment) waitForService(url string, timeout time.Duration) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		resp, err := client.Get(url)
+		if err == nil {
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK {
+				return true
+			}
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return false
+}
+
+func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string) string {
+	t.Helper()
+
+	configDir := filepath.Join(env.dataDir, "trino")
+	if err := os.MkdirAll(configDir, 0755); err != nil {
+		t.Fatalf("Failed to create Trino config dir: %v", err)
+	}
+
+	config := fmt.Sprintf(`connector.name=iceberg
+iceberg.catalog.type=rest
+iceberg.rest-catalog.uri=http://%s:%d
+iceberg.rest-catalog.warehouse=s3://%s/
+iceberg.file-format=PARQUET
+fs.native-s3.enabled=true
+s3.endpoint=http://%s:%d
+s3.path-style-access=true
+s3.aws-access-key=test
+s3.aws-secret-key=test
+s3.region=us-west-2
+`, env.bindIP, env.icebergPort, warehouseBucket, env.bindIP, env.s3Port)
+
+	if err := os.WriteFile(filepath.Join(configDir, "iceberg.properties"), []byte(config), 0644); err != nil {
+		t.Fatalf("Failed to write Trino config: %v", err)
+	}
+
+	return configDir
+}
+
+func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) {
+	t.Helper()
+
+	containerName := "seaweed-trino-" + randomString(8)
+	env.trinoContainer = containerName
+
+	cmd := exec.Command("docker", "run", "-d",
+		"--name", containerName,
+		"--add-host", "host.docker.internal:host-gateway",
+		"-v", fmt.Sprintf("%s:/etc/trino/catalog", configDir),
+		"-v", fmt.Sprintf("%s:/test", env.dataDir),
+		"-e", "AWS_ACCESS_KEY_ID=test",
+		"-e", "AWS_SECRET_ACCESS_KEY=test",
+		"-e", "AWS_REGION=us-west-2",
+		"trinodb/trino",
+	)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("Failed to start Trino container: %v\n%s", err, string(output))
+	}
+}
+
+func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	var lastOutput []byte
+	for time.Now().Before(deadline) {
+		cmd := exec.Command("docker", "exec", containerName,
+			"trino", "--catalog", "system", "--schema", "runtime",
+			"--execute", "SELECT 1",
+		)
+		if output, err := cmd.CombinedOutput(); err == nil {
+			return
+		} else {
+			lastOutput = output
+			outputStr := string(output)
+			if strings.Contains(outputStr, "No such container") ||
+				strings.Contains(outputStr, "is not running") {
+				break
+			}
+		}
+		time.Sleep(1 * time.Second)
+	}
+	logs, _ := exec.Command("docker", "logs", containerName).CombinedOutput()
+	t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s\nTrino logs:\n%s", string(lastOutput), string(logs))
+}
+
+func runTrinoSQL(t *testing.T, containerName, sql string) string {
+	t.Helper()
+
+	cmd := exec.Command("docker", "exec", containerName,
+		"trino", "--catalog", "system", "--schema", "runtime",
+		"--output-format", "CSV",
+		"--execute", sql,
+	)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, string(output))
+	}
+	return string(output)
+}
+
+func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	endpoint := fmt.Sprintf("http://%s:%d/buckets", env.bindIP, env.s3Port)
+	reqBody := fmt.Sprintf(`{"name":"%s"}`, bucketName)
+	req, err := http.NewRequest(http.MethodPut, endpoint, strings.NewReader(reqBody))
+	if err != nil {
+		t.Fatalf("Failed to create request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to create table bucket %s: %v", bucketName, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
+	}
+}
+
+func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	endpoint := fmt.Sprintf("http://%s:%d/%s", env.bindIP, env.s3Port, bucketName)
+	req, err := http.NewRequest(http.MethodPut, endpoint, nil)
+	if err != nil {
+		t.Fatalf("Failed to create S3 bucket request: %v", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to create S3 bucket %s: %v", bucketName, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("Failed to create S3 bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
+	}
+}
+
+func hasDocker() bool {
+	cmd := exec.Command("docker", "version")
+	return cmd.Run() == nil
+}
+
+func mustFreePort(t *testing.T, name string) int {
+	t.Helper()
+
+	port, err := getFreePort()
+	if err != nil {
+		t.Fatalf("Failed to get free port for %s: %v", name, err)
+	}
+	return port
+}
+
+func mustFreePortPair(t *testing.T, name string) (int, int) {
+	t.Helper()
+
+	httpPort, grpcPort, err := findAvailablePortPair()
+	if err != nil {
+		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
+	}
+	return httpPort, grpcPort
+}
+
+func findAvailablePortPair() (int, int, error) {
+	httpPort, err := getFreePort()
+	if err != nil {
+		return 0, 0, err
+	}
+	grpcPort, err := getFreePort()
+	if err != nil {
+		return 0, 0, err
+	}
+	return httpPort, grpcPort, nil
+}
+
+func getFreePort() (int, error) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return 0, err
+	}
+	defer listener.Close()
+
+	addr := listener.Addr().(*net.TCPAddr)
+	return addr.Port, nil
+}
+
+func findBindIP() string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "127.0.0.1"
+	}
+	for _, addr := range addrs {
+		ipNet, ok := addr.(*net.IPNet)
+		if !ok || ipNet.IP == nil {
+			continue
+		}
+		ip := ipNet.IP.To4()
+		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
+			continue
+		}
+		return ip.String()
+	}
+	return "127.0.0.1"
+}
+
+func randomString(length int) string {
+	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
+	b := make([]byte, length)
+	if _, err := rand.Read(b); err != nil {
+		panic("failed to generate random string: " + err.Error())
+	}
+	for i := range b {
+		b[i] = charset[int(b[i])%len(charset)]
+	}
+	return string(b)
+}

test/s3/etag/s3_etag_test.go

@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"io"
 	mathrand "math/rand"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -85,6 +86,12 @@ func init() {
 
 // getS3Client creates an AWS S3 v2 client for testing
 func getS3Client(t *testing.T) *s3.Client {
+	endpoint := os.Getenv("S3_ENDPOINT")
+	if endpoint == "" {
+		endpoint = defaultConfig.Endpoint
+	}
+	t.Logf("Using S3 endpoint: %s", endpoint)
+
 	cfg, err := config.LoadDefaultConfig(context.TODO(),
 		config.WithRegion(defaultConfig.Region),
 		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
@@ -95,7 +102,7 @@ func getS3Client(t *testing.T) *s3.Client {
 		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
 			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
 				return aws.Endpoint{
-					URL:               defaultConfig.Endpoint,
+					URL:               endpoint,
 					SigningRegion:     defaultConfig.Region,
 					HostnameImmutable: true,
 				}, nil
@@ -403,6 +410,101 @@ func TestMultipartUploadETagFormat(t *testing.T) {
 		"Part count in ETag should match number of parts uploaded")
 }
 
+func TestMultipartUploadETagVerification(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Create test data for multipart upload (11MB = 2 parts: 5MB + 6MB)
+	// Using parts of different sizes to ensure correct calculation
+	part1Size := 5 * 1024 * 1024
+	part2Size := 6 * 1024 * 1024
+	data1 := generateRandomData(part1Size)
+	data2 := generateRandomData(part2Size)
+
+	objectKey := "verify-etag-multipart.bin"
+
+	// Pre-calculate expected ETag
+	md1 := md5.Sum(data1)
+	md2 := md5.Sum(data2)
+	concatenatedMD5s := append(md1[:], md2[:]...)
+	finalMD5 := md5.Sum(concatenatedMD5s)
+	expectedETagValue := fmt.Sprintf("%x-2", finalMD5)
+
+	t.Logf("Expected multipart ETag: %s", expectedETagValue)
+
+	// 1. CreateMultipartUpload
+	createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	uploadId := createResp.UploadId
+
+	// 2. UploadPart 1
+	putPart1, err := client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   uploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(data1),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "\""+calculateMD5(data1)+"\"", aws.ToString(putPart1.ETag))
+
+	// 3. UploadPart 2
+	putPart2, err := client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   uploadId,
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(data2),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "\""+calculateMD5(data2)+"\"", aws.ToString(putPart2.ETag))
+
+	// 4. CompleteMultipartUpload
+	completeResp, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{ETag: putPart1.ETag, PartNumber: aws.Int32(1)},
+				{ETag: putPart2.ETag, PartNumber: aws.Int32(2)},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	completeETag := cleanETag(aws.ToString(completeResp.ETag))
+	t.Logf("CompleteMultipartUpload ETag: %s", completeETag)
+	assert.Equal(t, expectedETagValue, completeETag, "CompleteMultipartUpload ETag mismatch")
+
+	// 5. HeadObject
+	headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	headETag := cleanETag(aws.ToString(headResp.ETag))
+	assert.Equal(t, expectedETagValue, headETag, "HeadObject ETag mismatch")
+
+	// 6. GetObject
+	getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	getETag := cleanETag(aws.ToString(getResp.ETag))
+	assert.Equal(t, expectedETagValue, getETag, "GetObject ETag mismatch")
+	getResp.Body.Close()
+}
+
 // TestPutObjectETagConsistency verifies ETag consistency between PUT and GET
 func TestPutObjectETagConsistency(t *testing.T) {
 	ctx := context.Background()

test/s3/policy/policy_test.go

@@ -16,22 +16,25 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
 	"github.com/stretchr/testify/require"
 )
 
 // TestCluster manages the weed mini instance for integration testing
 type TestCluster struct {
-	dataDir    string
-	ctx        context.Context
-	cancel     context.CancelFunc
-	isRunning  bool
-	wg         sync.WaitGroup
-	masterPort int
-	volumePort int
-	filerPort  int
-	s3Port     int
-	s3Endpoint string
+	dataDir        string
+	ctx            context.Context
+	cancel         context.CancelFunc
+	isRunning      bool
+	wg             sync.WaitGroup
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	filerPort      int
+	filerGrpcPort  int
+	s3Port         int
+	s3Endpoint     string
 }
 
 func TestS3PolicyShellRevised(t *testing.T) {
@@ -53,8 +56,8 @@ func TestS3PolicyShellRevised(t *testing.T) {
 	require.NoError(t, tmpPolicyFile.Close())
 
 	weedCmd := "weed"
-	masterAddr := fmt.Sprintf("127.0.0.1:%d", cluster.masterPort)
-	filerAddr := fmt.Sprintf("127.0.0.1:%d", cluster.filerPort)
+	masterAddr := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filerAddr := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
 
 	// Put
 	execShell(t, weedCmd, masterAddr, filerAddr, fmt.Sprintf("s3.policy -put -name=testpolicy -file=%s", tmpPolicyFile.Name()))
@@ -156,17 +159,16 @@ func findAvailablePort() (int, error) {
 
 // findAvailablePortPair finds an available http port P such that P and P+10000 (grpc) are both available
 func findAvailablePortPair() (int, int, error) {
+	httpPort, err := findAvailablePort()
+	if err != nil {
+		return 0, 0, err
+	}
 	for i := 0; i < 100; i++ {
-		httpPort, err := findAvailablePort()
+		grpcPort, err := findAvailablePort()
 		if err != nil {
 			return 0, 0, err
 		}
-		grpcPort := httpPort + 10000
-
-		// check if grpc port is available
-		listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
-		if err == nil {
-			listener.Close()
+		if grpcPort != httpPort {
 			return httpPort, grpcPort, nil
 		}
 	}
@@ -188,14 +190,16 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	s3Endpoint := fmt.Sprintf("http://127.0.0.1:%d", s3Port)
 	cluster := &TestCluster{
-		dataDir:    testDir,
-		ctx:        ctx,
-		cancel:     cancel,
-		masterPort: masterPort,
-		volumePort: volumePort,
-		filerPort:  filerPort,
-		s3Port:     s3Port,
-		s3Endpoint: s3Endpoint,
+		dataDir:        testDir,
+		ctx:            ctx,
+		cancel:         cancel,
+		masterPort:     masterPort,
+		masterGrpcPort: masterGrpcPort,
+		volumePort:     volumePort,
+		filerPort:      filerPort,
+		filerGrpcPort:  filerGrpcPort,
+		s3Port:         s3Port,
+		s3Endpoint:     s3Endpoint,
 	}
 
 	// Disable authentication for tests

test/s3tables/catalog/iceberg_catalog_test.go

@@ -275,7 +275,7 @@ func TestIcebergNamespaces(t *testing.T) {
 	env.StartSeaweedFS(t)
 
 	// Create the default table bucket first via S3
-	createTableBucket(t, env, "default")
+	createTableBucket(t, env, "warehouse")
 
 	// Test GET /v1/namespaces (should return empty list initially)
 	resp, err := http.Get(env.IcebergURL() + "/v1/namespaces")

test/s3tables/catalog/test_rest_catalog.py

@@ -188,16 +188,30 @@ def main():
     print(f"Prefix: {args.prefix}")
     print()
     
-    # Load the REST catalog
-    catalog = load_catalog(
-        "rest",
-        **{
-            "type": "rest",
-            "uri": args.catalog_url,
-            "warehouse": args.warehouse,
-            "prefix": args.prefix,
-        }
-    )
+    # Load the REST catalog with retries to handle possible delay in catalog server readiness
+    import time
+    max_retries = 10
+    catalog = None
+    for attempt in range(max_retries):
+        try:
+            catalog = load_catalog(
+                "rest",
+                **{
+                    "type": "rest",
+                    "uri": args.catalog_url,
+                    "warehouse": args.warehouse,
+                    "prefix": args.prefix,
+                }
+            )
+            print(f"Successfully connected to catalog on attempt {attempt + 1}")
+            break
+        except Exception as e:
+            if attempt < max_retries - 1:
+                print(f"  Attempt {attempt + 1} failed, retrying in 2s... ({e})")
+                time.sleep(2)
+            else:
+                print(f"  All {max_retries} attempts failed.")
+                raise e
     
     # Run tests
     tests = [

test/s3tables/catalog_trino/trino_catalog_test.go

@@ -0,0 +1,542 @@
+package catalog_trino
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type TestEnvironment struct {
+	seaweedDir      string
+	weedBinary      string
+	dataDir         string
+	bindIP          string
+	s3Port          int
+	s3GrpcPort      int
+	icebergPort     int
+	masterPort      int
+	masterGrpcPort  int
+	filerPort       int
+	filerGrpcPort   int
+	volumePort      int
+	volumeGrpcPort  int
+	weedProcess     *exec.Cmd
+	weedCancel      context.CancelFunc
+	trinoContainer  string
+	dockerAvailable bool
+	accessKey       string
+	secretKey       string
+}
+
+func TestTrinoIcebergCatalog(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Trino integration test")
+	}
+
+	fmt.Printf(">>> Starting SeaweedFS...\n")
+	env.StartSeaweedFS(t)
+	fmt.Printf(">>> SeaweedFS started.\n")
+
+	catalogBucket := "warehouse"
+	tableBucket := "iceberg-tables"
+	fmt.Printf(">>> Creating table bucket: %s\n", tableBucket)
+	createTableBucket(t, env, tableBucket)
+	fmt.Printf(">>> Creating table bucket: %s\n", catalogBucket)
+	createTableBucket(t, env, catalogBucket)
+	fmt.Printf(">>> All buckets created.\n")
+
+	// Test Iceberg REST API directly
+	testIcebergRestAPI(t, env)
+
+	configDir := env.writeTrinoConfig(t, catalogBucket)
+	env.startTrinoContainer(t, configDir)
+	waitForTrino(t, env.trinoContainer, 60*time.Second)
+
+	schemaName := "trino_" + randomString(6)
+
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, schemaName) {
+		t.Fatalf("Expected schema %s in output:\n%s", schemaName, output)
+	}
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SHOW TABLES FROM iceberg.%s", schemaName))
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	info, err := os.Stat(weedBinary)
+	if err != nil || info.IsDir() {
+		// Try looking for weed/weed/weed
+		weedBinary = filepath.Join(seaweedDir, "weed", "weed", "weed")
+		info, err = os.Stat(weedBinary)
+		if err != nil || info.IsDir() {
+			weedBinary = "weed"
+			if _, err := exec.LookPath(weedBinary); err != nil {
+				t.Skip("weed binary not found, skipping integration test")
+			}
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-trino-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+
+	bindIP := findBindIP()
+
+	masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := mustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := mustFreePortPair(t, "S3")
+	icebergPort := mustFreePort(t, "Iceberg")
+
+	return &TestEnvironment{
+		seaweedDir:      seaweedDir,
+		weedBinary:      weedBinary,
+		dataDir:         dataDir,
+		bindIP:          bindIP,
+		s3Port:          s3Port,
+		s3GrpcPort:      s3GrpcPort,
+		icebergPort:     icebergPort,
+		masterPort:      masterPort,
+		masterGrpcPort:  masterGrpcPort,
+		filerPort:       filerPort,
+		filerGrpcPort:   filerGrpcPort,
+		volumePort:      volumePort,
+		volumeGrpcPort:  volumeGrpcPort,
+		dockerAvailable: hasDocker(),
+		accessKey:       "AKIAIOSFODNN7EXAMPLE",
+		secretKey:       "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+	}
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	// Create IAM config file
+	iamConfigPath := filepath.Join(env.dataDir, "iam_config.json")
+	iamConfig := fmt.Sprintf(`{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "%s",
+          "secretKey": "%s"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ]
+}`, env.accessKey, env.secretKey)
+
+	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
+		t.Fatalf("Failed to create IAM config: %v", err)
+	}
+
+	securityToml := filepath.Join(env.dataDir, "security.toml")
+	if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
+		t.Fatalf("Failed to create security.toml: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
+		"-s3.config", iamConfigPath,
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	// Set AWS credentials in environment (for compatibility)
+	cmd.Env = append(os.Environ(),
+		"AWS_ACCESS_KEY_ID="+env.accessKey,
+		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
+	)
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
+	}
+	env.weedProcess = cmd
+
+	// Try to check if Iceberg API is ready
+	// First try checking the /v1/config endpoint (requires auth, so will return 401 if server is up)
+	icebergURL := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
+	if !env.waitForService(icebergURL, 30*time.Second) {
+		// Try to get more info about why it failed
+		client := &http.Client{Timeout: 2 * time.Second}
+		resp, err := client.Get(icebergURL)
+		if err != nil {
+			t.Logf("WARNING: Could not connect to Iceberg service at %s: %v", icebergURL, err)
+		} else {
+			t.Logf("WARNING: Iceberg service returned status %d at %s", resp.StatusCode, icebergURL)
+			resp.Body.Close()
+		}
+		t.Fatalf("Iceberg REST API did not become ready")
+	}
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+
+	if env.trinoContainer != "" {
+		_ = exec.Command("docker", "rm", "-f", env.trinoContainer).Run()
+	}
+
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+
+	if env.weedProcess != nil {
+		time.Sleep(2 * time.Second)
+		_ = env.weedProcess.Wait()
+	}
+
+	if env.dataDir != "" {
+		_ = os.RemoveAll(env.dataDir)
+	}
+}
+
+func (env *TestEnvironment) waitForService(url string, timeout time.Duration) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		resp, err := client.Get(url)
+		if err != nil {
+			// Service not responding yet
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+		statusCode := resp.StatusCode
+		resp.Body.Close()
+		// Accept 2xx status codes (successful responses)
+		if statusCode >= 200 && statusCode < 300 {
+			return true
+		}
+		// Also accept 401/403 (auth errors mean service is up, just needs credentials)
+		if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
+			return true
+		}
+		// For other status codes, keep trying
+		time.Sleep(500 * time.Millisecond)
+	}
+	return false
+}
+
+func testIcebergRestAPI(t *testing.T, env *TestEnvironment) {
+	t.Helper()
+	fmt.Printf(">>> Testing Iceberg REST API directly...\n")
+
+	// First, verify the service is listening
+	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", env.bindIP, env.icebergPort))
+	if err != nil {
+		t.Fatalf("Cannot connect to Iceberg service at %s:%d: %v", env.bindIP, env.icebergPort, err)
+	}
+	conn.Close()
+	t.Logf("Successfully connected to Iceberg service at %s:%d", env.bindIP, env.icebergPort)
+
+	// Test /v1/config endpoint
+	url := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
+	t.Logf("Testing Iceberg REST API at %s", url)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatalf("Failed to connect to Iceberg REST API at %s: %v", url, err)
+	}
+	defer resp.Body.Close()
+
+	t.Logf("Iceberg REST API response status: %d", resp.StatusCode)
+	body, _ := io.ReadAll(resp.Body)
+	t.Logf("Iceberg REST API response body: %s", string(body))
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Expected 200 OK from /v1/config, got %d", resp.StatusCode)
+	}
+}
+
+func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string) string {
+	t.Helper()
+
+	configDir := filepath.Join(env.dataDir, "trino")
+	if err := os.MkdirAll(configDir, 0755); err != nil {
+		t.Fatalf("Failed to create Trino config dir: %v", err)
+	}
+
+	config := fmt.Sprintf(`connector.name=iceberg
+iceberg.catalog.type=rest
+iceberg.rest-catalog.uri=http://host.docker.internal:%d
+iceberg.rest-catalog.warehouse=s3://%s/
+iceberg.file-format=PARQUET
+
+# S3 storage config
+fs.native-s3.enabled=true
+s3.endpoint=http://host.docker.internal:%d
+s3.path-style-access=true
+s3.signer-type=AwsS3V4Signer
+s3.aws-access-key=%s
+s3.aws-secret-key=%s
+s3.region=us-west-2
+
+# REST catalog authentication
+iceberg.rest-catalog.security=SIGV4
+`, env.icebergPort, warehouseBucket, env.s3Port, env.accessKey, env.secretKey)
+
+	if err := os.WriteFile(filepath.Join(configDir, "iceberg.properties"), []byte(config), 0644); err != nil {
+		t.Fatalf("Failed to write Trino config: %v", err)
+	}
+
+	return configDir
+}
+
+func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) {
+	t.Helper()
+
+	containerName := "seaweed-trino-" + randomString(8)
+	env.trinoContainer = containerName
+
+	cmd := exec.Command("docker", "run", "-d",
+		"--name", containerName,
+		"--add-host", "host.docker.internal:host-gateway",
+		"-v", fmt.Sprintf("%s:/etc/trino/catalog", configDir),
+		"-v", fmt.Sprintf("%s:/test", env.dataDir),
+		"-e", "AWS_ACCESS_KEY_ID="+env.accessKey,
+		"-e", "AWS_SECRET_ACCESS_KEY="+env.secretKey,
+		"-e", "AWS_REGION=us-west-2",
+		"trinodb/trino:479",
+	)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("Failed to start Trino container: %v\n%s", err, string(output))
+	}
+}
+
+func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	var lastOutput []byte
+	retryCount := 0
+	for time.Now().Before(deadline) {
+		// Try system catalog query as a readiness check
+		cmd := exec.Command("docker", "exec", containerName,
+			"trino", "--catalog", "system", "--schema", "runtime",
+			"--execute", "SELECT 1",
+		)
+		if output, err := cmd.CombinedOutput(); err == nil {
+			return
+		} else {
+			lastOutput = output
+			outputStr := string(output)
+			if strings.Contains(outputStr, "No such container") ||
+				strings.Contains(outputStr, "is not running") {
+				break
+			}
+			retryCount++
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	// If we can't connect to system catalog, try to at least connect to Trino server
+	cmd := exec.Command("docker", "exec", containerName, "trino", "--version")
+	if err := cmd.Run(); err == nil {
+		// Trino process is running, even if catalog isn't ready yet
+		// Give it a bit more time
+		time.Sleep(5 * time.Second)
+		return
+	}
+
+	logs, _ := exec.Command("docker", "logs", containerName).CombinedOutput()
+	t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s\nTrino logs:\n%s", string(lastOutput), string(logs))
+}
+
+func runTrinoSQL(t *testing.T, containerName, sql string) string {
+	t.Helper()
+
+	cmd := exec.Command("docker", "exec", containerName,
+		"trino", "--catalog", "iceberg",
+		"--output-format", "CSV",
+		"--execute", sql,
+	)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		logs, _ := exec.Command("docker", "logs", containerName).CombinedOutput()
+		t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s\nTrino logs:\n%s", err, sql, string(output), string(logs))
+	}
+	return string(output)
+}
+
+func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	// Use weed shell to create the table bucket
+	// Create with "000000000000" account ID (matches AccountAdmin.Id from auth_credentials.go)
+	// This ensures bucket owner matches authenticated identity's Account.Id
+	cmd := exec.Command(env.weedBinary, "shell",
+		fmt.Sprintf("-master=%s:%d.%d", env.bindIP, env.masterPort, env.masterGrpcPort),
+	)
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
+	fmt.Printf(">>> EXECUTING: %v\n", cmd.Args)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Printf(">>> ERROR Output: %s\n", string(output))
+		t.Fatalf("Failed to create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, string(output))
+	}
+	fmt.Printf(">>> SUCCESS: Created table bucket %s\n", bucketName)
+
+	t.Logf("Created table bucket: %s", bucketName)
+}
+
+func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	// Create an AWS S3 client with the test credentials pointing to our local server
+	cfg := aws.Config{
+		Region:       "us-east-1",
+		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
+		BaseEndpoint: aws.String(fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port)),
+	}
+
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+
+	// Create the bucket using standard S3 API
+	_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create object bucket %s: %v", bucketName, err)
+	}
+}
+
+func hasDocker() bool {
+	cmd := exec.Command("docker", "version")
+	return cmd.Run() == nil
+}
+
+func mustFreePort(t *testing.T, name string) int {
+	t.Helper()
+
+	port, err := getFreePort()
+	if err != nil {
+		t.Fatalf("Failed to get free port for %s: %v", name, err)
+	}
+	return port
+}
+
+func mustFreePortPair(t *testing.T, name string) (int, int) {
+	t.Helper()
+
+	httpPort, grpcPort, err := findAvailablePortPair()
+	if err != nil {
+		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
+	}
+	return httpPort, grpcPort
+}
+
+func findAvailablePortPair() (int, int, error) {
+	httpPort, err := getFreePort()
+	if err != nil {
+		return 0, 0, err
+	}
+	grpcPort, err := getFreePort()
+	if err != nil {
+		return 0, 0, err
+	}
+	return httpPort, grpcPort, nil
+}
+
+func getFreePort() (int, error) {
+	listener, err := net.Listen("tcp", "0.0.0.0:0")
+	if err != nil {
+		return 0, err
+	}
+	defer listener.Close()
+
+	addr := listener.Addr().(*net.TCPAddr)
+	return addr.Port, nil
+}
+
+func findBindIP() string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "127.0.0.1"
+	}
+	for _, addr := range addrs {
+		ipNet, ok := addr.(*net.IPNet)
+		if !ok || ipNet.IP == nil {
+			continue
+		}
+		ip := ipNet.IP.To4()
+		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
+			continue
+		}
+		return ip.String()
+	}
+	return "127.0.0.1"
+}
+
+func randomString(length int) string {
+	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
+	b := make([]byte, length)
+	if _, err := rand.Read(b); err != nil {
+		panic("failed to generate random string: " + err.Error())
+	}
+	for i := range b {
+		b[i] = charset[int(b[i])%len(charset)]
+	}
+	return string(b)
+}

test/s3tables/catalog_trino/trino_crud_operations_test.go

@@ -0,0 +1,209 @@
+package catalog_trino
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+// setupTrinoTest is a helper function that sets up the common test environment for all Trino CRUD tests
+func setupTrinoTest(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Trino integration test")
+	}
+
+	t.Logf(">>> Starting SeaweedFS...")
+	env.StartSeaweedFS(t)
+
+	catalogBucket := "warehouse"
+	tableBucket := "iceberg-tables"
+	createTableBucket(t, env, tableBucket)
+	createTableBucket(t, env, catalogBucket)
+
+	configDir := env.writeTrinoConfig(t, catalogBucket)
+	env.startTrinoContainer(t, configDir)
+	waitForTrino(t, env.trinoContainer, 60*time.Second)
+
+	return env
+}
+
+// TestNamespaceCRUD tests namespace (schema) CRUD operations via Trino SQL
+// Namespaces are the key container for tables in Iceberg, and this test
+// verifies the full CRUD lifecycle: Create, Read (List), Update, and Delete
+func TestNamespaceCRUD(t *testing.T) {
+	env := setupTrinoTest(t)
+	defer env.Cleanup(t)
+
+	// CREATE: Create a namespace
+	namespace1 := "crud_test_ns1_" + randomString(6)
+	t.Logf(">>> CREATE: Creating namespace %s", namespace1)
+	runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+namespace1)
+	t.Logf(">>> Namespace %s created", namespace1)
+
+	namespace2 := "crud_test_ns2_" + randomString(6)
+	t.Logf(">>> CREATE: Creating second namespace %s", namespace2)
+	runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+namespace2)
+	t.Logf(">>> Namespace %s created", namespace2)
+
+	// READ: List all namespaces
+	t.Logf(">>> READ: Listing all namespaces")
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, namespace1) {
+		t.Fatalf("Expected namespace %s in listing output:\n%s", namespace1, output)
+	}
+	if !strings.Contains(output, namespace2) {
+		t.Fatalf("Expected namespace %s in listing output:\n%s", namespace2, output)
+	}
+	t.Logf(">>> Both namespaces found in listing")
+
+	// UPDATE: Namespaces typically don't have "update" semantics in SQL,
+	// but we can verify properties via metadata queries
+	t.Logf(">>> UPDATE: Simulating namespace properties (via SQL properties check)")
+	// Iceberg REST API supports namespace properties, but Trino SQL doesn't expose them directly
+	// This test verifies the namespace still exists and is accessible
+	output = runTrinoSQL(t, env.trinoContainer, "SHOW TABLES FROM iceberg."+namespace1)
+
+	// DELETE: Drop namespaces
+	t.Logf(">>> DELETE: Dropping namespace %s", namespace1)
+	runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+namespace1)
+	t.Logf(">>> Namespace %s dropped", namespace1)
+
+	t.Logf(">>> DELETE: Dropping namespace %s", namespace2)
+	runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+namespace2)
+	t.Logf(">>> Namespace %s dropped", namespace2)
+
+	// Verify deletion: Namespaces should no longer exist
+	t.Logf(">>> VERIFY: Checking that namespaces are deleted")
+	output = runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if strings.Contains(output, namespace1) {
+		t.Errorf("Namespace %s still appears in listing after deletion", namespace1)
+	} else {
+		t.Logf(">>> Namespace %s correctly deleted", namespace1)
+	}
+	if strings.Contains(output, namespace2) {
+		t.Errorf("Namespace %s still appears in listing after deletion", namespace2)
+	} else {
+		t.Logf(">>> Namespace %s correctly deleted", namespace2)
+	}
+
+	t.Logf(">>> TestNamespaceCRUD PASSED")
+}
+
+// TestNamespaceListingPagination tests that namespace listing works correctly
+// This verifies the LIST operation with multiple namespaces
+func TestNamespaceListingPagination(t *testing.T) {
+	env := setupTrinoTest(t)
+	defer env.Cleanup(t)
+
+	// Create multiple namespaces
+	numNamespaces := 5
+	namespaces := make([]string, numNamespaces)
+	t.Logf(">>> Creating %d namespaces for listing test", numNamespaces)
+	for i := 0; i < numNamespaces; i++ {
+		namespaces[i] = "list_test_ns" + fmt.Sprintf("%d", i+1) + "_" + randomString(4)
+		runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+namespaces[i])
+		t.Logf(">>> Created namespace %d: %s", i+1, namespaces[i])
+	}
+
+	// List all namespaces
+	t.Logf(">>> Listing all namespaces")
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+
+	// Verify all namespaces are in the listing
+	t.Logf(">>> Verifying all namespaces are in listing")
+	for i, ns := range namespaces {
+		if !strings.Contains(output, ns) {
+			t.Fatalf("Expected namespace %d (%s) in listing output", i+1, ns)
+		}
+		t.Logf(">>> Namespace %d (%s) found in listing", i+1, ns)
+	}
+
+	// Clean up
+	t.Logf(">>> Cleaning up namespaces")
+	for _, ns := range namespaces {
+		runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+ns)
+	}
+	t.Logf(">>> Cleanup complete")
+
+	t.Logf(">>> TestNamespaceListingPagination PASSED")
+}
+
+// TestNamespaceErrorHandling tests error handling for namespace operations
+// Tests both idempotent operations and actual error cases
+func TestNamespaceErrorHandling(t *testing.T) {
+	env := setupTrinoTest(t)
+	defer env.Cleanup(t)
+
+	// Test 1: Create and drop same namespace
+	ns := "error_test_ns_" + randomString(6)
+	t.Logf(">>> Test 1: Creating and dropping namespace %s", ns)
+	runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+ns)
+	runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+ns)
+	t.Logf(">>> Namespace created and dropped successfully")
+
+	// Test 2: Try to drop non-existent namespace with IF EXISTS (should succeed gracefully)
+	nonExistent := "nonexistent_" + randomString(6)
+	t.Logf(">>> Test 2: Attempting to drop non-existent namespace %s with IF EXISTS", nonExistent)
+	runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA IF EXISTS iceberg."+nonExistent)
+	t.Logf(">>> Drop non-existent namespace handled gracefully (IF EXISTS clause)")
+
+	// Test 3: Creating duplicate namespace (with IF NOT EXISTS - should succeed gracefully)
+	ns2 := "dup_test_ns_" + randomString(6)
+	t.Logf(">>> Test 3: Creating namespace %s twice (using IF NOT EXISTS)", ns2)
+	runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+ns2)
+	runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+ns2)
+	t.Logf(">>> Duplicate creation handled gracefully")
+
+	// Test 4: Verify schema properties persist after creation
+	t.Logf(">>> Test 4: Verifying namespace still exists after duplicate creation attempt")
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, ns2) {
+		t.Errorf("Expected namespace %s to exist in listing after duplicate creation attempt", ns2)
+	} else {
+		t.Logf(">>> Namespace %s correctly persists", ns2)
+	}
+
+	// Cleanup
+	runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+ns2)
+
+	t.Logf(">>> TestNamespaceErrorHandling PASSED")
+}
+
+// TestSchemaIntegrationWithCatalog tests that schemas are properly integrated with the catalog
+// This verifies that schema operations through Trino are visible through Iceberg REST API concepts
+func TestSchemaIntegrationWithCatalog(t *testing.T) {
+	env := setupTrinoTest(t)
+	defer env.Cleanup(t)
+
+	// Create a schema through Trino
+	schemaName := "catalog_integration_" + randomString(6)
+	t.Logf(">>> Creating schema through Trino SQL: %s", schemaName)
+	runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+schemaName)
+
+	// Verify it's accessible through catalog (list schemas)
+	t.Logf(">>> Verifying schema is visible through catalog (SHOW SCHEMAS)")
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, schemaName) {
+		t.Fatalf("Created schema %s not visible in catalog listing", schemaName)
+	}
+	t.Logf(">>> Schema successfully verified in catalog")
+
+	// Verify empty schema
+	t.Logf(">>> Verifying schema is empty (no tables)")
+	output = runTrinoSQL(t, env.trinoContainer, "SHOW TABLES FROM iceberg."+schemaName)
+
+	// Clean up
+	t.Logf(">>> Cleaning up schema")
+	runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+schemaName)
+
+	t.Logf(">>> TestSchemaIntegrationWithCatalog PASSED")
+}

test/s3tables/table-buckets/s3tables_integration_test.go

@@ -480,53 +480,51 @@ func testTargetOperations(t *testing.T, client *S3TablesClient) {
 
 // Helper functions
 
-// findAvailablePort finds an available port by binding to port 0
-func findAvailablePort() (int, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
+// findAvailablePorts finds n available ports by binding to port 0 multiple times
+// It keeps the listeners open until all ports are found to ensure uniqueness
+func findAvailablePorts(n int) ([]int, error) {
+	listeners := make([]*net.TCPListener, n)
+	ports := make([]int, n)
 
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
+	// Open all listeners to ensure we get unique ports
+	for i := 0; i < n; i++ {
+		listener, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			// Close valid listeners before returning error
+			for j := 0; j < i; j++ {
+				listeners[j].Close()
+			}
+			return nil, err
+		}
+		listeners[i] = listener.(*net.TCPListener)
+		ports[i] = listeners[i].Addr().(*net.TCPAddr).Port
+	}
+
+	// Close all listeners
+	for _, l := range listeners {
+		l.Close()
+	}
+
+	return ports, nil
 }
 
 // startMiniCluster starts a weed mini instance directly without exec
 func startMiniCluster(t *testing.T) (*TestCluster, error) {
 	// Find available ports
-	masterPort, err := findAvailablePort()
+	// We need 8 unique ports: Master(2), Volume(2), Filer(2), S3(2)
+	ports, err := findAvailablePorts(8)
 	if err != nil {
-		return nil, fmt.Errorf("failed to find master port: %v", err)
-	}
-	masterGrpcPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find master grpc port: %v", err)
-	}
-	volumePort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find volume port: %v", err)
-	}
-	volumeGrpcPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find volume grpc port: %v", err)
-	}
-	filerPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find filer port: %v", err)
-	}
-	filerGrpcPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find filer grpc port: %v", err)
-	}
-	s3Port, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find s3 port: %v", err)
-	}
-	s3GrpcPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find s3 grpc port: %v", err)
+		return nil, fmt.Errorf("failed to find available ports: %v", err)
 	}
+
+	masterPort := ports[0]
+	masterGrpcPort := ports[1]
+	volumePort := ports[2]
+	volumeGrpcPort := ports[3]
+	filerPort := ports[4]
+	filerGrpcPort := ports[5]
+	s3Port := ports[6]
+	s3GrpcPort := ports[7]
 	// Create temporary directory for test data
 	testDir := t.TempDir()
 

unmaintained/local-nginx/nginx.conf

@@ -12,14 +12,25 @@ http {
     sendfile        on;
     keepalive_timeout  65;
 
+    log_format debug '$remote_addr - $remote_user [$time_local] '
+                      '"$request" $status $body_bytes_sent '
+                      'request_body: "$request_body" '
+                      'upstream_status: "$upstream_status" '
+                      'upstream_response_time: "$upstream_response_time" '
+                      'resp_content_type: "$sent_http_content_type" '
+                      'resp_content_length: "$sent_http_content_length"';
+
+    access_log logs/access.log debug;
+    error_log logs/error.log debug;
+
     # Standard Configuration from Wiki
     upstream seaweedfs {
-        server 127.0.0.1:8334;
+        server 127.0.0.1:8333;
         keepalive 20;
     }
 
     server {
-        listen       8333 ssl;
+        listen       443 ssl;
         server_name  localhost;
 
         ssl_certificate      cert.pem;
@@ -33,6 +44,7 @@ http {
 
         ignore_invalid_headers off;
         client_max_body_size 0;
+        client_body_buffer_size 128k;
         proxy_buffering off;
 
         location / {
@@ -48,8 +60,12 @@ http {
             proxy_connect_timeout 300;
             proxy_http_version 1.1;
             proxy_set_header Connection "";
-            proxy_request_buffering off;
+            proxy_request_buffering on;
             chunked_transfer_encoding off;
+
+            # Log response headers
+            add_header X-Debug-Status $upstream_status;
+            add_header X-Debug-Time $upstream_response_time;
         }
     }
 }

weed/admin/dash/admin_server.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
@@ -33,6 +32,17 @@ import (
 	_ "github.com/seaweedfs/seaweedfs/weed/credential/grpc" // Register gRPC credential store
 )
 
+const (
+	maxAssignmentHistoryDisplay = 50
+	maxLogMessageLength         = 2000
+	maxLogFields                = 20
+	maxRelatedTasksDisplay      = 50
+	maxRecentTasksDisplay       = 10
+	defaultCacheTimeout         = 10 * time.Second
+	defaultFilerCacheTimeout    = 30 * time.Second
+	defaultStatsCacheTimeout    = 30 * time.Second
+)
+
 // FilerConfig holds filer configuration needed for bucket operations
 type FilerConfig struct {
 	BucketsPath string
@@ -132,10 +142,10 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		templateFS:                    templateFS,
 		dataDir:                       dataDir,
 		grpcDialOption:                grpcDialOption,
-		cacheExpiration:               10 * time.Second,
-		filerCacheExpiration:          30 * time.Second, // Cache filers for 30 seconds
+		cacheExpiration:               defaultCacheTimeout,
+		filerCacheExpiration:          defaultFilerCacheTimeout,
 		configPersistence:             NewConfigPersistence(dataDir),
-		collectionStatsCacheThreshold: 30 * time.Second,
+		collectionStatsCacheThreshold: defaultStatsCacheTimeout,
 		s3TablesManager:               newS3TablesManager(),
 		icebergPort:                   icebergPort,
 	}
@@ -779,7 +789,7 @@ func (s *AdminServer) GetClusterBrokers() (*ClusterBrokersData, error) {
 
 // ShowMaintenanceQueue displays the maintenance queue page
 func (as *AdminServer) ShowMaintenanceQueue(c *gin.Context) {
-	data, err := as.getMaintenanceQueueData()
+	data, err := as.GetMaintenanceQueueData()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -868,7 +878,7 @@ func (as *AdminServer) TriggerMaintenanceScan(c *gin.Context) {
 
 // GetMaintenanceTasks returns all maintenance tasks
 func (as *AdminServer) GetMaintenanceTasks(c *gin.Context) {
-	tasks, err := as.getMaintenanceTasks()
+	tasks, err := as.GetAllMaintenanceTasks()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -1032,9 +1042,9 @@ func (as *AdminServer) UpdateMaintenanceConfigData(config *maintenance.Maintenan
 
 // Helper methods for maintenance operations
 
-// getMaintenanceQueueData returns data for the maintenance queue UI
-func (as *AdminServer) getMaintenanceQueueData() (*maintenance.MaintenanceQueueData, error) {
-	tasks, err := as.getMaintenanceTasks()
+// GetMaintenanceQueueData returns data for the maintenance queue UI
+func (as *AdminServer) GetMaintenanceQueueData() (*maintenance.MaintenanceQueueData, error) {
+	tasks, err := as.GetAllMaintenanceTasks()
 	if err != nil {
 		return nil, err
 	}
@@ -1089,14 +1099,16 @@ func (as *AdminServer) getMaintenanceQueueStats() (*maintenance.QueueStats, erro
 	return queueStats, nil
 }
 
-// getMaintenanceTasks returns all maintenance tasks
-func (as *AdminServer) getMaintenanceTasks() ([]*maintenance.MaintenanceTask, error) {
+// GetAllMaintenanceTasks returns all maintenance tasks
+func (as *AdminServer) GetAllMaintenanceTasks() ([]*maintenance.MaintenanceTask, error) {
 	if as.maintenanceManager == nil {
 		return []*maintenance.MaintenanceTask{}, nil
 	}
 
-	// Collect all tasks from memory across all statuses
-	allTasks := []*maintenance.MaintenanceTask{}
+	// 1. Collect all tasks from memory
+	tasksMap := make(map[string]*maintenance.MaintenanceTask)
+
+	// Collect from memory via GetTasks loop to ensure we catch everything
 	statuses := []maintenance.MaintenanceTaskStatus{
 		maintenance.TaskStatusPending,
 		maintenance.TaskStatusAssigned,
@@ -1108,29 +1120,92 @@ func (as *AdminServer) getMaintenanceTasks() ([]*maintenance.MaintenanceTask, er
 
 	for _, status := range statuses {
 		tasks := as.maintenanceManager.GetTasks(status, "", 0)
-		allTasks = append(allTasks, tasks...)
+		for _, t := range tasks {
+			tasksMap[t.ID] = t
+		}
 	}
 
-	// Also load any persisted tasks that might not be in memory
+	// 2. Merge persisted tasks
 	if as.configPersistence != nil {
 		persistedTasks, err := as.configPersistence.LoadAllTaskStates()
 		if err == nil {
-			// Add any persisted tasks not already in memory
-			for _, persistedTask := range persistedTasks {
-				found := false
-				for _, memoryTask := range allTasks {
-					if memoryTask.ID == persistedTask.ID {
-						found = true
-						break
-					}
-				}
-				if !found {
-					allTasks = append(allTasks, persistedTask)
+			for _, t := range persistedTasks {
+				if _, exists := tasksMap[t.ID]; !exists {
+					tasksMap[t.ID] = t
 				}
 			}
 		}
 	}
 
+	// 3. Bucketize buckets
+	var pendingTasks, activeTasks, finishedTasks []*maintenance.MaintenanceTask
+
+	for _, t := range tasksMap {
+		switch t.Status {
+		case maintenance.TaskStatusPending:
+			pendingTasks = append(pendingTasks, t)
+		case maintenance.TaskStatusAssigned, maintenance.TaskStatusInProgress:
+			activeTasks = append(activeTasks, t)
+		case maintenance.TaskStatusCompleted, maintenance.TaskStatusFailed, maintenance.TaskStatusCancelled:
+			finishedTasks = append(finishedTasks, t)
+		default:
+			// Treat unknown as finished/archived? Or pending?
+			// Safest to add to finished so they appear somewhere
+			finishedTasks = append(finishedTasks, t)
+		}
+	}
+
+	// 4. Sort buckets
+	// Pending: Newest Created First
+	sort.Slice(pendingTasks, func(i, j int) bool {
+		return pendingTasks[i].CreatedAt.After(pendingTasks[j].CreatedAt)
+	})
+
+	// Active: Newest Created First (or StartedAt?)
+	sort.Slice(activeTasks, func(i, j int) bool {
+		return activeTasks[i].CreatedAt.After(activeTasks[j].CreatedAt)
+	})
+
+	// Finished: Newest Completed First
+	sort.Slice(finishedTasks, func(i, j int) bool {
+		t1 := finishedTasks[i].CompletedAt
+		t2 := finishedTasks[j].CompletedAt
+
+		// Handle nil completion times
+		if t1 == nil && t2 == nil {
+			// Both nil, fallback to CreatedAt
+			if !finishedTasks[i].CreatedAt.Equal(finishedTasks[j].CreatedAt) {
+				return finishedTasks[i].CreatedAt.After(finishedTasks[j].CreatedAt)
+			}
+			return finishedTasks[i].ID > finishedTasks[j].ID
+		}
+		if t1 == nil {
+			return false // t1 (nil) goes to bottom
+		}
+		if t2 == nil {
+			return true // t2 (nil) goes to bottom
+		}
+
+		// Compare completion times
+		if !t1.Equal(*t2) {
+			return t1.After(*t2)
+		}
+
+		// Fallback to CreatedAt if completion times are identical
+		if !finishedTasks[i].CreatedAt.Equal(finishedTasks[j].CreatedAt) {
+			return finishedTasks[i].CreatedAt.After(finishedTasks[j].CreatedAt)
+		}
+
+		// Final tie-breaker: ID
+		return finishedTasks[i].ID > finishedTasks[j].ID
+	})
+
+	// 5. Recombine
+	allTasks := make([]*maintenance.MaintenanceTask, 0, len(tasksMap))
+	allTasks = append(allTasks, pendingTasks...)
+	allTasks = append(allTasks, activeTasks...)
+	allTasks = append(allTasks, finishedTasks...)
+
 	return allTasks, nil
 }
 
@@ -1181,15 +1256,25 @@ func (as *AdminServer) GetMaintenanceTaskDetail(taskID string) (*maintenance.Tas
 		return nil, err
 	}
 
+	// Copy task and truncate assignment history for display
+	displayTask := *task
+	displayTask.AssignmentHistory = nil // History is provided separately in taskDetail
+
 	// Create task detail structure from the loaded task
 	taskDetail := &maintenance.TaskDetailData{
-		Task:              task,
+		Task:              &displayTask,
 		AssignmentHistory: task.AssignmentHistory, // Use assignment history from persisted task
 		ExecutionLogs:     []*maintenance.TaskExecutionLog{},
 		RelatedTasks:      []*maintenance.MaintenanceTask{},
 		LastUpdated:       time.Now(),
 	}
 
+	// Truncate assignment history if it's too long (display last N only)
+	if len(taskDetail.AssignmentHistory) > maxAssignmentHistoryDisplay {
+		startIdx := len(taskDetail.AssignmentHistory) - maxAssignmentHistoryDisplay
+		taskDetail.AssignmentHistory = taskDetail.AssignmentHistory[startIdx:]
+	}
+
 	if taskDetail.AssignmentHistory == nil {
 		taskDetail.AssignmentHistory = []*maintenance.TaskAssignmentRecord{}
 	}
@@ -1205,72 +1290,19 @@ func (as *AdminServer) GetMaintenanceTaskDetail(taskID string) (*maintenance.Tas
 		}
 	}
 
-	// Get execution logs from worker if task is active/completed and worker is connected
-	if task.Status == maintenance.TaskStatusInProgress || task.Status == maintenance.TaskStatusCompleted {
-		if as.workerGrpcServer != nil && task.WorkerID != "" {
-			workerLogs, err := as.workerGrpcServer.RequestTaskLogs(task.WorkerID, taskID, 100, "")
-			if err == nil && len(workerLogs) > 0 {
-				// Convert worker logs to maintenance logs
-				for _, workerLog := range workerLogs {
-					maintenanceLog := &maintenance.TaskExecutionLog{
-						Timestamp: time.Unix(workerLog.Timestamp, 0),
-						Level:     workerLog.Level,
-						Message:   workerLog.Message,
-						Source:    "worker",
-						TaskID:    taskID,
-						WorkerID:  task.WorkerID,
-					}
-					// carry structured fields if present
-					if len(workerLog.Fields) > 0 {
-						maintenanceLog.Fields = make(map[string]string, len(workerLog.Fields))
-						for k, v := range workerLog.Fields {
-							maintenanceLog.Fields[k] = v
-						}
-					}
-					// carry optional progress/status
-					if workerLog.Progress != 0 {
-						p := float64(workerLog.Progress)
-						maintenanceLog.Progress = &p
-					}
-					if workerLog.Status != "" {
-						maintenanceLog.Status = workerLog.Status
-					}
-					taskDetail.ExecutionLogs = append(taskDetail.ExecutionLogs, maintenanceLog)
-				}
-			} else if err != nil {
-				// Add a diagnostic log entry when worker logs cannot be retrieved
-				diagnosticLog := &maintenance.TaskExecutionLog{
-					Timestamp: time.Now(),
-					Level:     "WARNING",
-					Message:   fmt.Sprintf("Failed to retrieve worker logs: %v", err),
-					Source:    "admin",
-					TaskID:    taskID,
-					WorkerID:  task.WorkerID,
-				}
-				taskDetail.ExecutionLogs = append(taskDetail.ExecutionLogs, diagnosticLog)
-				glog.V(1).Infof("Failed to get worker logs for task %s from worker %s: %v", taskID, task.WorkerID, err)
-			}
+	// Load execution logs from disk
+	if as.configPersistence != nil {
+		logs, err := as.configPersistence.LoadTaskExecutionLogs(taskID)
+		if err == nil {
+			taskDetail.ExecutionLogs = logs
 		} else {
-			// Add diagnostic information when worker is not available
-			reason := "worker gRPC server not available"
-			if task.WorkerID == "" {
-				reason = "no worker assigned to task"
-			}
-			diagnosticLog := &maintenance.TaskExecutionLog{
-				Timestamp: time.Now(),
-				Level:     "INFO",
-				Message:   fmt.Sprintf("Worker logs not available: %s", reason),
-				Source:    "admin",
-				TaskID:    taskID,
-				WorkerID:  task.WorkerID,
-			}
-			taskDetail.ExecutionLogs = append(taskDetail.ExecutionLogs, diagnosticLog)
+			glog.V(2).Infof("No execution logs found on disk for task %s", taskID)
 		}
 	}
 
 	// Get related tasks (other tasks on same volume/server)
 	if task.VolumeID != 0 || task.Server != "" {
-		allTasks := as.maintenanceManager.GetTasks("", "", 50) // Get recent tasks
+		allTasks := as.maintenanceManager.GetTasks("", "", maxRelatedTasksDisplay) // Get recent tasks
 		for _, relatedTask := range allTasks {
 			if relatedTask.ID != taskID &&
 				(relatedTask.VolumeID == task.VolumeID || relatedTask.Server == task.Server) {
@@ -1324,7 +1356,7 @@ func (as *AdminServer) getMaintenanceWorkerDetails(workerID string) (*WorkerDeta
 	}
 
 	// Get recent tasks for this worker
-	recentTasks := as.maintenanceManager.GetTasks(TaskStatusCompleted, "", 10)
+	recentTasks := as.maintenanceManager.GetTasks(TaskStatusCompleted, "", maxRecentTasksDisplay)
 	var workerRecentTasks []*MaintenanceTask
 	for _, task := range recentTasks {
 		if task.WorkerID == workerID {
@@ -1336,12 +1368,13 @@ func (as *AdminServer) getMaintenanceWorkerDetails(workerID string) (*WorkerDeta
 	var totalDuration time.Duration
 	var completedTasks, failedTasks int
 	for _, task := range workerRecentTasks {
-		if task.Status == TaskStatusCompleted {
+		switch task.Status {
+		case TaskStatusCompleted:
 			completedTasks++
 			if task.StartedAt != nil && task.CompletedAt != nil {
 				totalDuration += task.CompletedAt.Sub(*task.StartedAt)
 			}
-		} else if task.Status == TaskStatusFailed {
+		case TaskStatusFailed:
 			failedTasks++
 		}
 	}
@@ -1370,31 +1403,29 @@ func (as *AdminServer) getMaintenanceWorkerDetails(workerID string) (*WorkerDeta
 	}, nil
 }
 
-// GetWorkerLogs fetches logs from a specific worker for a task
+// GetWorkerLogs fetches logs from a specific worker for a task (now reads from disk)
 func (as *AdminServer) GetWorkerLogs(c *gin.Context) {
 	workerID := c.Param("id")
 	taskID := c.Query("taskId")
-	maxEntriesStr := c.DefaultQuery("maxEntries", "100")
-	logLevel := c.DefaultQuery("logLevel", "")
 
-	maxEntries := int32(100)
-	if maxEntriesStr != "" {
-		if parsed, err := strconv.ParseInt(maxEntriesStr, 10, 32); err == nil {
-			maxEntries = int32(parsed)
-		}
-	}
-
-	if as.workerGrpcServer == nil {
-		c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Worker gRPC server not available"})
+	// Check config persistence first
+	if as.configPersistence == nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Config persistence not available"})
 		return
 	}
 
-	logs, err := as.workerGrpcServer.RequestTaskLogs(workerID, taskID, maxEntries, logLevel)
+	// Load logs strictly from disk to avoid timeouts and network dependency
+	// This matches the behavior of the Task Detail page
+	logs, err := as.configPersistence.LoadTaskExecutionLogs(taskID)
 	if err != nil {
-		c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("Failed to get logs from worker: %v", err)})
-		return
+		glog.V(2).Infof("No execution logs found on disk for task %s: %v", taskID, err)
+		logs = []*maintenance.TaskExecutionLog{}
 	}
 
+	// Filter logs by workerID if strictly needed, but usually task logs are what we want
+	// The persistent logs struct (TaskExecutionLog) matches what the frontend expects for the detail view
+	// ensuring consistent display.
+
 	c.JSON(http.StatusOK, gin.H{"worker_id": workerID, "task_id": taskID, "logs": logs, "count": len(logs)})
 }
 

weed/admin/dash/config_persistence.go

@@ -962,7 +962,36 @@ func (cp *ConfigPersistence) CleanupCompletedTasks() error {
 
 	// Sort by completion time (most recent first)
 	sort.Slice(completedTasks, func(i, j int) bool {
-		return completedTasks[i].CompletedAt.After(*completedTasks[j].CompletedAt)
+		t1 := completedTasks[i].CompletedAt
+		t2 := completedTasks[j].CompletedAt
+
+		// Handle nil completion times
+		if t1 == nil && t2 == nil {
+			// Both nil, fallback to CreatedAt
+			if !completedTasks[i].CreatedAt.Equal(completedTasks[j].CreatedAt) {
+				return completedTasks[i].CreatedAt.After(completedTasks[j].CreatedAt)
+			}
+			return completedTasks[i].ID > completedTasks[j].ID
+		}
+		if t1 == nil {
+			return false // t1 (nil) goes to bottom
+		}
+		if t2 == nil {
+			return true // t2 (nil) goes to bottom
+		}
+
+		// Compare completion times
+		if !t1.Equal(*t2) {
+			return t1.After(*t2)
+		}
+
+		// Fallback to CreatedAt if completion times are identical
+		if !completedTasks[i].CreatedAt.Equal(completedTasks[j].CreatedAt) {
+			return completedTasks[i].CreatedAt.After(completedTasks[j].CreatedAt)
+		}
+
+		// Final tie-breaker: ID
+		return completedTasks[i].ID > completedTasks[j].ID
 	})
 
 	// Keep only the most recent MaxCompletedTasks, delete the rest

weed/admin/dash/worker_grpc_server.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
@@ -17,6 +18,15 @@ import (
 	"google.golang.org/grpc/peer"
 )
 
+const (
+	maxLogFetchLimit   = 1000
+	maxLogMessageSize  = 2000
+	maxLogFieldsCount  = 20
+	logRequestTimeout  = 10 * time.Second
+	logResponseTimeout = 30 * time.Second
+	logSendTimeout     = 10 * time.Second
+)
+
 // WorkerGrpcServer implements the WorkerService gRPC interface
 type WorkerGrpcServer struct {
 	worker_pb.UnimplementedWorkerServiceServer
@@ -42,7 +52,6 @@ type LogRequestContext struct {
 	TaskID     string
 	WorkerID   string
 	ResponseCh chan *worker_pb.TaskLogResponse
-	Timeout    time.Time
 }
 
 // WorkerConnection represents an active worker connection
@@ -89,8 +98,9 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
 	s.listener = listener
 	s.running = true
 
-	// Start cleanup routine
+	// Start background routines
 	go s.cleanupRoutine()
+	go s.activeLogFetchLoop()
 
 	// Start serving in a goroutine
 	go func() {
@@ -408,7 +418,7 @@ func (s *WorkerGrpcServer) handleTaskRequest(conn *WorkerConnection, request *wo
 
 		select {
 		case conn.outgoing <- noTaskAssignment:
-			glog.V(2).Infof("Sent 'No Task' response to worker %s", conn.workerID)
+			glog.V(4).Infof("Sent 'No Task' response to worker %s", conn.workerID)
 		case <-time.After(time.Second):
 			// If we can't send, the worker will eventually time out and reconnect, which is fine
 		}
@@ -437,9 +447,90 @@ func (s *WorkerGrpcServer) handleTaskCompletion(conn *WorkerConnection, completi
 		} else {
 			glog.Errorf("Worker %s failed task %s: %s", conn.workerID, completion.TaskId, completion.ErrorMessage)
 		}
+
+		// Fetch and persist logs
+		go s.FetchAndSaveLogs(conn.workerID, completion.TaskId)
 	}
 }
 
+// FetchAndSaveLogs retrieves logs from a worker and saves them to disk
+func (s *WorkerGrpcServer) FetchAndSaveLogs(workerID, taskID string) error {
+	// Add a small initial delay to allow worker to finalize and sync logs
+	// especially when this is called immediately after TaskComplete
+	time.Sleep(300 * time.Millisecond)
+
+	var workerLogs []*worker_pb.TaskLogEntry
+	var err error
+
+	// Retry a few times if fetch fails, as logs might be in the middle of a terminal sync
+	for attempt := 1; attempt <= 3; attempt++ {
+		workerLogs, err = s.RequestTaskLogs(workerID, taskID, maxLogFetchLimit, "")
+		if err == nil {
+			break
+		}
+		if attempt < 3 {
+			glog.V(1).Infof("Fetch logs attempt %d failed for task %s: %v. Retrying in 1s...", attempt, taskID, err)
+			time.Sleep(1 * time.Second)
+		}
+	}
+
+	if err != nil {
+		glog.Warningf("Failed to fetch logs for task %s after 3 attempts: %v", taskID, err)
+		return err
+	}
+
+	// Convert logs
+	var maintenanceLogs []*maintenance.TaskExecutionLog
+	for _, workerLog := range workerLogs {
+		maintenanceLog := &maintenance.TaskExecutionLog{
+			Timestamp: time.Unix(workerLog.Timestamp, 0),
+			Level:     workerLog.Level,
+			Message:   workerLog.Message,
+			Source:    "worker",
+			TaskID:    taskID,
+			WorkerID:  workerID,
+		}
+
+		// Truncate very long messages to prevent rendering issues and disk bloat
+		if len(maintenanceLog.Message) > maxLogMessageSize {
+			maintenanceLog.Message = maintenanceLog.Message[:maxLogMessageSize] + "... (truncated)"
+		}
+
+		// carry structured fields if present
+		if len(workerLog.Fields) > 0 {
+			maintenanceLog.Fields = make(map[string]string)
+			fieldCount := 0
+			for k, v := range workerLog.Fields {
+				if fieldCount >= maxLogFieldsCount {
+					maintenanceLog.Fields["..."] = fmt.Sprintf("(%d more fields truncated)", len(workerLog.Fields)-maxLogFieldsCount)
+					break
+				}
+				maintenanceLog.Fields[k] = v
+				fieldCount++
+			}
+		}
+
+		// carry optional progress/status
+		if workerLog.Progress != 0 {
+			p := float64(workerLog.Progress)
+			maintenanceLog.Progress = &p
+		}
+		if workerLog.Status != "" {
+			maintenanceLog.Status = workerLog.Status
+		}
+		maintenanceLogs = append(maintenanceLogs, maintenanceLog)
+	}
+
+	// Persist logs
+	if s.adminServer.configPersistence != nil {
+		if err := s.adminServer.configPersistence.SaveTaskExecutionLogs(taskID, maintenanceLogs); err != nil {
+			glog.Errorf("Failed to persist logs for task %s: %v", taskID, err)
+			return err
+		}
+	}
+	return nil
+}
+
 // handleTaskLogResponse processes task log responses from workers
 func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, response *worker_pb.TaskLogResponse) {
 	requestKey := fmt.Sprintf("%s:%s", response.WorkerId, response.TaskId)
@@ -575,10 +666,13 @@ func (s *WorkerGrpcServer) RequestTaskLogs(workerID, taskID string, maxEntries i
 		TaskID:     taskID,
 		WorkerID:   workerID,
 		ResponseCh: responseCh,
-		Timeout:    time.Now().Add(10 * time.Second),
 	}
 
 	s.logRequestsMutex.Lock()
+	if _, exists := s.pendingLogRequests[requestKey]; exists {
+		s.logRequestsMutex.Unlock()
+		return nil, fmt.Errorf("a log request for task %s is already in progress", taskID)
+	}
 	s.pendingLogRequests[requestKey] = requestContext
 	s.logRequestsMutex.Unlock()
 
@@ -601,10 +695,12 @@ func (s *WorkerGrpcServer) RequestTaskLogs(workerID, taskID string, maxEntries i
 	select {
 	case conn.outgoing <- logRequest:
 		glog.V(1).Infof("Log request sent to worker %s for task %s", workerID, taskID)
-	case <-time.After(5 * time.Second):
+	case <-time.After(logSendTimeout):
 		// Clean up pending request on timeout
 		s.logRequestsMutex.Lock()
-		delete(s.pendingLogRequests, requestKey)
+		if s.pendingLogRequests[requestKey] == requestContext {
+			delete(s.pendingLogRequests, requestKey)
+		}
 		s.logRequestsMutex.Unlock()
 		return nil, fmt.Errorf("timeout sending log request to worker %s", workerID)
 	}
@@ -617,10 +713,12 @@ func (s *WorkerGrpcServer) RequestTaskLogs(workerID, taskID string, maxEntries i
 		}
 		glog.V(1).Infof("Received %d log entries for task %s from worker %s", len(response.LogEntries), taskID, workerID)
 		return response.LogEntries, nil
-	case <-time.After(10 * time.Second):
+	case <-time.After(logResponseTimeout):
 		// Clean up pending request on timeout
 		s.logRequestsMutex.Lock()
-		delete(s.pendingLogRequests, requestKey)
+		if s.pendingLogRequests[requestKey] == requestContext {
+			delete(s.pendingLogRequests, requestKey)
+		}
 		s.logRequestsMutex.Unlock()
 		return nil, fmt.Errorf("timeout waiting for log response from worker %s", workerID)
 	}
@@ -684,3 +782,38 @@ func findClientAddress(ctx context.Context) string {
 	}
 	return pr.Addr.String()
 }
+
+// activeLogFetchLoop periodically fetches logs for all in-progress tasks
+func (s *WorkerGrpcServer) activeLogFetchLoop() {
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-s.stopChan:
+			return
+		case <-ticker.C:
+			if !s.running || s.adminServer == nil || s.adminServer.maintenanceManager == nil {
+				continue
+			}
+
+			// Get all in-progress tasks
+			tasks := s.adminServer.maintenanceManager.GetTasks(maintenance.TaskStatusInProgress, "", 0)
+			if len(tasks) == 0 {
+				continue
+			}
+
+			glog.V(2).Infof("Background log fetcher: found %d in-progress tasks", len(tasks))
+			for _, task := range tasks {
+				if task.WorkerID != "" {
+					// Use a goroutine to avoid blocking the loop
+					go func(wID, tID string) {
+						if err := s.FetchAndSaveLogs(wID, tID); err != nil {
+							glog.V(2).Infof("Background log fetch failed for task %s on worker %s: %v", tID, wID, err)
+						}
+					}(task.WorkerID, task.ID)
+				}
+			}
+		}
+	}
+}

weed/admin/handlers/maintenance_handlers.go

@@ -39,6 +39,11 @@ func NewMaintenanceHandlers(adminServer *dash.AdminServer) *MaintenanceHandlers
 func (h *MaintenanceHandlers) ShowTaskDetail(c *gin.Context) {
 	taskID := c.Param("id")
 
+	if h.adminServer == nil {
+		c.String(http.StatusInternalServerError, "Admin server not initialized")
+		return
+	}
+
 	taskDetail, err := h.adminServer.GetMaintenanceTaskDetail(taskID)
 	if err != nil {
 		glog.Errorf("DEBUG ShowTaskDetail: error getting task detail for %s: %v", taskID, err)
@@ -111,6 +116,10 @@ func (h *MaintenanceHandlers) ShowMaintenanceQueue(c *gin.Context) {
 
 // ShowMaintenanceWorkers displays the maintenance workers page
 func (h *MaintenanceHandlers) ShowMaintenanceWorkers(c *gin.Context) {
+	if h.adminServer == nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Admin server not initialized"})
+		return
+	}
 	workersData, err := h.adminServer.GetMaintenanceWorkersData()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
@@ -339,6 +348,8 @@ func (h *MaintenanceHandlers) UpdateTaskConfig(c *gin.Context) {
 			glog.Warningf("Failed to save task config to protobuf file: %v", err)
 			// Don't fail the request, just log the warning
 		}
+	} else if h.adminServer == nil {
+		glog.Warningf("Failed to save task config: admin server not initialized")
 	}
 
 	// Trigger a configuration reload in the maintenance manager
@@ -492,74 +503,25 @@ func (h *MaintenanceHandlers) UpdateMaintenanceConfig(c *gin.Context) {
 // Helper methods that delegate to AdminServer
 
 func (h *MaintenanceHandlers) getMaintenanceQueueData() (*maintenance.MaintenanceQueueData, error) {
-	tasks, err := h.getMaintenanceTasks()
-	if err != nil {
-		return nil, err
-	}
-
-	workers, err := h.getMaintenanceWorkers()
-	if err != nil {
-		return nil, err
-	}
-
-	stats, err := h.getMaintenanceQueueStats()
-	if err != nil {
-		return nil, err
-	}
-
-	data := &maintenance.MaintenanceQueueData{
-		Tasks:       tasks,
-		Workers:     workers,
-		Stats:       stats,
-		LastUpdated: time.Now(),
-	}
-
-	return data, nil
-}
-
-func (h *MaintenanceHandlers) getMaintenanceQueueStats() (*maintenance.QueueStats, error) {
-	// Use the exported method from AdminServer
-	return h.adminServer.GetMaintenanceQueueStats()
-}
-
-func (h *MaintenanceHandlers) getMaintenanceTasks() ([]*maintenance.MaintenanceTask, error) {
-	// Call the maintenance manager directly to get recent tasks (limit for performance)
 	if h.adminServer == nil {
-		return []*maintenance.MaintenanceTask{}, nil
+		return nil, fmt.Errorf("admin server not initialized")
 	}
-
-	manager := h.adminServer.GetMaintenanceManager()
-	if manager == nil {
-		return []*maintenance.MaintenanceTask{}, nil
-	}
-
-	// Get recent tasks only (last 100) to prevent slow page loads
-	// Users can view more tasks via pagination if needed
-	allTasks := manager.GetTasks("", "", 100)
-	return allTasks, nil
-}
-
-func (h *MaintenanceHandlers) getMaintenanceWorkers() ([]*maintenance.MaintenanceWorker, error) {
-	// Get workers from the admin server's maintenance manager
-	if h.adminServer == nil {
-		return []*maintenance.MaintenanceWorker{}, nil
-	}
-
-	if h.adminServer.GetMaintenanceManager() == nil {
-		return []*maintenance.MaintenanceWorker{}, nil
-	}
-
-	// Get workers from the maintenance manager
-	workers := h.adminServer.GetMaintenanceManager().GetWorkers()
-	return workers, nil
+	// Use the exported method from AdminServer used by the JSON API
+	return h.adminServer.GetMaintenanceQueueData()
 }
 
 func (h *MaintenanceHandlers) getMaintenanceConfig() (*maintenance.MaintenanceConfigData, error) {
+	if h.adminServer == nil {
+		return nil, fmt.Errorf("admin server not initialized")
+	}
 	// Delegate to AdminServer's real persistence method
 	return h.adminServer.GetMaintenanceConfigData()
 }
 
 func (h *MaintenanceHandlers) updateMaintenanceConfig(config *maintenance.MaintenanceConfig) error {
+	if h.adminServer == nil {
+		return fmt.Errorf("admin server not initialized")
+	}
 	// Delegate to AdminServer's real persistence method
 	return h.adminServer.UpdateMaintenanceConfigData(config)
 }

weed/admin/maintenance/maintenance_integration.go

@@ -305,6 +305,7 @@ func (s *MaintenanceIntegration) convertToExistingFormat(result *types.TaskDetec
 	}
 
 	return &TaskDetectionResult{
+		TaskID:      result.TaskID,
 		TaskType:    existingType,
 		VolumeID:    result.VolumeID,
 		Server:      result.Server,
@@ -523,19 +524,25 @@ func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
 	var estimatedSize int64
 
 	if task.TypedParams != nil {
+		// Calculate storage impact for this task type
+		// Volume size is not currently used for Balance/Vacuum impact and is not stored in MaintenanceTask
+		sourceImpact, targetImpact := topology.CalculateTaskStorageImpact(topology.TaskType(string(taskType)), 0)
+
 		// Use unified sources and targets from TaskParams
 		for _, src := range task.TypedParams.Sources {
 			sources = append(sources, topology.TaskSource{
-				SourceServer: src.Node,
-				SourceDisk:   src.DiskId,
+				SourceServer:  src.Node,
+				SourceDisk:    src.DiskId,
+				StorageChange: sourceImpact,
 			})
 			// Sum estimated size from all sources
 			estimatedSize += int64(src.EstimatedSize)
 		}
 		for _, target := range task.TypedParams.Targets {
 			destinations = append(destinations, topology.TaskDestination{
-				TargetServer: target.Node,
-				TargetDisk:   target.DiskId,
+				TargetServer:  target.Node,
+				TargetDisk:    target.DiskId,
+				StorageChange: targetImpact,
 			})
 		}
 

weed/admin/maintenance/maintenance_manager.go

@@ -603,6 +603,13 @@ func (mm *MaintenanceManager) CancelTask(taskID string) error {
 			}
 		}
 
+		// Notify ActiveTopology to release capacity
+		if mm.scanner != nil && mm.scanner.integration != nil {
+			if at := mm.scanner.integration.GetActiveTopology(); at != nil {
+				_ = at.CompleteTask(taskID)
+			}
+		}
+
 		glog.V(2).Infof("Cancelled task %s", taskID)
 		return nil
 	}

weed/admin/maintenance/maintenance_queue.go

@@ -90,6 +90,11 @@ func (mq *MaintenanceQueue) LoadTasksFromPersistence() error {
 				}
 			}
 		}
+
+		// Sync task with ActiveTopology for capacity tracking
+		if mq.integration != nil {
+			mq.integration.SyncTask(task)
+		}
 	}
 
 	// Sort pending tasks by priority and schedule time
@@ -134,7 +139,9 @@ func (mq *MaintenanceQueue) AddTask(task *MaintenanceTask) {
 		return
 	}
 
-	task.ID = generateTaskID()
+	if task.ID == "" {
+		task.ID = generateTaskID()
+	}
 	task.Status = TaskStatusPending
 	task.CreatedAt = time.Now()
 	task.MaxRetries = 3 // Default retry count
@@ -200,6 +207,7 @@ func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult)
 		}
 
 		task := &MaintenanceTask{
+			ID:         result.TaskID,
 			Type:       result.TaskType,
 			Priority:   result.Priority,
 			VolumeID:   result.VolumeID,
@@ -272,7 +280,7 @@ func (mq *MaintenanceQueue) GetNextTask(workerID string, capabilities []Maintena
 
 	// If no task found, return nil
 	if selectedTask == nil {
-		glog.V(2).Infof("No suitable tasks available for worker %s (checked %d pending tasks)", workerID, len(mq.pendingTasks))
+		glog.V(4).Infof("No suitable tasks available for worker %s (checked %d pending tasks)", workerID, len(mq.pendingTasks))
 		return nil
 	}
 
@@ -311,6 +319,24 @@ func (mq *MaintenanceQueue) GetNextTask(workerID string, capabilities []Maintena
 	selectedTask.WorkerID = workerID
 	selectedTask.StartedAt = &now
 
+	// Notify ActiveTopology to reserve capacity (move from pending to assigned)
+	if mq.integration != nil {
+		if at := mq.integration.GetActiveTopology(); at != nil {
+			if err := at.AssignTask(selectedTask.ID); err != nil {
+				glog.Warningf("Failed to update ActiveTopology for task assignment %s: %v. Rolling back assignment.", selectedTask.ID, err)
+				// Rollback assignment in MaintenanceQueue
+				selectedTask.Status = TaskStatusPending
+				selectedTask.WorkerID = ""
+				selectedTask.StartedAt = nil
+				if len(selectedTask.AssignmentHistory) > 0 {
+					selectedTask.AssignmentHistory = selectedTask.AssignmentHistory[:len(selectedTask.AssignmentHistory)-1]
+				}
+				// Return nil so the task is not removed from pendingTasks and not returned to the worker
+				return nil
+			}
+		}
+	}
+
 	// Remove from pending tasks
 	mq.pendingTasks = append(mq.pendingTasks[:selectedIndex], mq.pendingTasks[selectedIndex+1:]...)
 
@@ -342,6 +368,17 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 		return
 	}
 
+	// Notify ActiveTopology to release capacity (move from assigned to recent)
+	// We do this for both success and failure cases to release the capacity
+	if mq.integration != nil {
+		if at := mq.integration.GetActiveTopology(); at != nil {
+			if task.Status == TaskStatusAssigned || task.Status == TaskStatusInProgress {
+				// Ignore error as task might not be in ActiveTopology (e.g. after restart)
+				_ = at.CompleteTask(taskID)
+			}
+		}
+	}
+
 	completedTime := time.Now()
 	task.CompletedAt = &completedTime
 
@@ -377,6 +414,12 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 			task.ScheduledAt = time.Now().Add(15 * time.Minute) // Retry delay
 
 			mq.pendingTasks = append(mq.pendingTasks, task)
+
+			// Resync with ActiveTopology (re-add as pending)
+			if mq.integration != nil {
+				mq.integration.SyncTask(task)
+			}
+
 			// Save task state after retry setup
 			mq.saveTaskState(task)
 			glog.Warningf("Task failed, scheduling retry: %s (%s) attempt %d/%d, worker %s, duration %v, error: %s",
@@ -587,15 +630,35 @@ func (mq *MaintenanceQueue) GetTasks(status MaintenanceTaskStatus, taskType Main
 			continue
 		}
 		tasks = append(tasks, task)
-		if limit > 0 && len(tasks) >= limit {
-			break
-		}
 	}
 
-	// Sort by creation time (newest first)
-	sort.Slice(tasks, func(i, j int) bool {
-		return tasks[i].CreatedAt.After(tasks[j].CreatedAt)
-	})
+	// Sort based on status
+	if status == TaskStatusCompleted || status == TaskStatusFailed || status == TaskStatusCancelled {
+		sort.Slice(tasks, func(i, j int) bool {
+			t1 := tasks[i].CompletedAt
+			t2 := tasks[j].CompletedAt
+			if t1 == nil && t2 == nil {
+				return tasks[i].CreatedAt.After(tasks[j].CreatedAt)
+			}
+			if t1 == nil {
+				return false
+			}
+			if t2 == nil {
+				return true
+			}
+			return t1.After(*t2)
+		})
+	} else {
+		// Default to creation time (newest first)
+		sort.Slice(tasks, func(i, j int) bool {
+			return tasks[i].CreatedAt.After(tasks[j].CreatedAt)
+		})
+	}
+
+	// Apply limit after sorting
+	if limit > 0 && len(tasks) > limit {
+		tasks = tasks[:limit]
+	}
 
 	return tasks
 }
@@ -683,6 +746,13 @@ func (mq *MaintenanceQueue) RemoveStaleWorkers(timeout time.Duration) int {
 					task.Error = "Worker became unavailable"
 					completedTime := time.Now()
 					task.CompletedAt = &completedTime
+
+					// Notify ActiveTopology to release capacity
+					if mq.integration != nil {
+						if at := mq.integration.GetActiveTopology(); at != nil {
+							_ = at.CompleteTask(task.ID)
+						}
+					}
 				}
 			}
 

weed/admin/maintenance/maintenance_queue_test.go

@@ -2,7 +2,10 @@ package maintenance
 
 import (
 	"testing"
+	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 )
 
@@ -351,3 +354,618 @@ func TestCanScheduleTaskNow_WithPolicy(t *testing.T) {
 		t.Errorf("Expected canScheduleTaskNow to return false when at policy limit, got true")
 	}
 }
+
+func TestMaintenanceQueue_TaskIDPreservation(t *testing.T) {
+	// Setup Policy
+	policy := &MaintenancePolicy{
+		TaskPolicies:        make(map[string]*worker_pb.TaskPolicy),
+		GlobalMaxConcurrent: 10,
+	}
+
+	// Setup Queue and Integration
+	mq := NewMaintenanceQueue(policy)
+	// We handle the integration manually to avoid complex setup
+	// integration := NewMaintenanceIntegration(mq, policy)
+	// mq.SetIntegration(integration)
+
+	// 2. Verify ID Preservation in AddTasksFromResults
+	originalID := "ec_task_123"
+	results := []*TaskDetectionResult{
+		{
+			TaskID:      originalID,
+			TaskType:    MaintenanceTaskType("erasure_coding"),
+			VolumeID:    100,
+			Server:      "server1",
+			Priority:    PriorityNormal,
+			TypedParams: &worker_pb.TaskParams{},
+		},
+	}
+
+	mq.AddTasksFromResults(results)
+
+	// Verify task exists with correct ID
+	queuedTask, exists := mq.tasks[originalID]
+	if !exists {
+		t.Errorf("Task with original ID %s not found in queue", originalID)
+	} else {
+		if queuedTask.ID != originalID {
+			t.Errorf("Task ID mismatch: expected %s, got %s", originalID, queuedTask.ID)
+		}
+	}
+
+	// 3. Verify AddTask preserves ID
+	manualTask := &MaintenanceTask{
+		ID:     "manual_id_456",
+		Type:   MaintenanceTaskType("vacuum"),
+		Status: TaskStatusPending,
+	}
+	mq.AddTask(manualTask)
+
+	if manualTask.ID != "manual_id_456" {
+		t.Errorf("AddTask overwrote ID: expected manual_id_456, got %s", manualTask.ID)
+	}
+}
+
+func TestMaintenanceQueue_ActiveTopologySync(t *testing.T) {
+	// Setup Policy
+	policy := &MaintenancePolicy{
+		TaskPolicies: map[string]*worker_pb.TaskPolicy{
+			"balance": {MaxConcurrent: 1},
+		},
+		GlobalMaxConcurrent: 10,
+	}
+
+	// Setup Queue and Integration
+	mq := NewMaintenanceQueue(policy)
+	integration := NewMaintenanceIntegration(mq, policy)
+	mq.SetIntegration(integration)
+
+	// 4. Verify ActiveTopology Synchronization (Assign and Complete)
+	// Get and Setup Topology
+	at := integration.GetActiveTopology()
+	if at == nil {
+		t.Fatalf("ActiveTopology not found in integration")
+	}
+
+	topologyInfo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "server1",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {
+										DiskId:         1,
+										VolumeCount:    1,
+										MaxVolumeCount: 10,
+										VolumeInfos: []*master_pb.VolumeInformationMessage{
+											{Id: 100, Collection: "col1"},
+										},
+									},
+									"hdd2": {
+										DiskId:         2,
+										VolumeCount:    0,
+										MaxVolumeCount: 10,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	at.UpdateTopology(topologyInfo)
+
+	// Add pending task to ActiveTopology
+	taskID := "sync_test_123"
+	err := at.AddPendingTask(topology.TaskSpec{
+		TaskID:     taskID,
+		TaskType:   topology.TaskTypeBalance,
+		VolumeID:   100,
+		VolumeSize: 1024 * 1024,
+		Sources: []topology.TaskSourceSpec{
+			{ServerID: "server1", DiskID: 1},
+		},
+		Destinations: []topology.TaskDestinationSpec{
+			{ServerID: "server1", DiskID: 2},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to add pending task to ActiveTopology: %v", err)
+	}
+
+	// Add the same task to MaintenanceQueue
+	mq.AddTask(&MaintenanceTask{
+		ID:         taskID,
+		Type:       MaintenanceTaskType("balance"),
+		VolumeID:   100,
+		Server:     "server1",
+		Collection: "col1",
+		TypedParams: &worker_pb.TaskParams{
+			TaskId: taskID,
+			Targets: []*worker_pb.TaskTarget{
+				{Node: "server1", DiskId: 2},
+			},
+		},
+	})
+
+	// Check initial available capacity on destination disk (server1:2)
+	// server1:2 has MaxVolumeCount=10, VolumeCount=0.
+	// Capacity should be 9 because AddPendingTask already reserved 1 slot.
+	capacityBefore := at.GetEffectiveAvailableCapacity("server1", 2)
+	if capacityBefore != 9 {
+		t.Errorf("Expected capacity 9 after AddPendingTask, got %d", capacityBefore)
+	}
+
+	// 5. Verify AssignTask (via GetNextTask)
+	mq.workers["worker1"] = &MaintenanceWorker{
+		ID:            "worker1",
+		Status:        "active",
+		Capabilities:  []MaintenanceTaskType{"balance"},
+		MaxConcurrent: 10,
+	}
+
+	taskFound := mq.GetNextTask("worker1", []MaintenanceTaskType{"balance"})
+	if taskFound == nil || taskFound.ID != taskID {
+		t.Fatalf("Expected to get task %s, got %+v", taskID, taskFound)
+	}
+
+	// Capacity should still be 9 on destination disk (server1:2)
+	capacityAfterAssign := at.GetEffectiveAvailableCapacity("server1", 2)
+	if capacityAfterAssign != 9 {
+		t.Errorf("Capacity should still be 9 after assignment, got %d", capacityAfterAssign)
+	}
+
+	// 6. Verify CompleteTask
+	mq.CompleteTask(taskID, "")
+
+	// Capacity should be released back to 10
+	capacityAfterComplete := at.GetEffectiveAvailableCapacity("server1", 2)
+	if capacityAfterComplete != 10 {
+		t.Errorf("Capacity should have returned to 10 after completion, got %d", capacityAfterComplete)
+	}
+}
+
+func TestMaintenanceQueue_StaleWorkerCapacityRelease(t *testing.T) {
+	// Setup
+	policy := &MaintenancePolicy{
+		TaskPolicies: map[string]*worker_pb.TaskPolicy{
+			"balance": {MaxConcurrent: 1},
+		},
+	}
+	mq := NewMaintenanceQueue(policy)
+	integration := NewMaintenanceIntegration(mq, policy)
+	mq.SetIntegration(integration)
+	at := integration.GetActiveTopology()
+
+	topologyInfo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "server1",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd1": {DiskId: 1, VolumeCount: 1, MaxVolumeCount: 10},
+									"hdd2": {DiskId: 2, VolumeCount: 0, MaxVolumeCount: 10},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	at.UpdateTopology(topologyInfo)
+
+	taskID := "stale_test_123"
+	at.AddPendingTask(topology.TaskSpec{
+		TaskID:       taskID,
+		TaskType:     topology.TaskTypeBalance,
+		VolumeID:     100,
+		VolumeSize:   1024,
+		Sources:      []topology.TaskSourceSpec{{ServerID: "server1", DiskID: 1}},
+		Destinations: []topology.TaskDestinationSpec{{ServerID: "server1", DiskID: 2}},
+	})
+
+	mq.AddTask(&MaintenanceTask{
+		ID:       taskID,
+		Type:     "balance",
+		VolumeID: 100,
+		Server:   "server1",
+		TypedParams: &worker_pb.TaskParams{
+			TaskId:  taskID,
+			Targets: []*worker_pb.TaskTarget{{Node: "server1", DiskId: 2}},
+		},
+	})
+
+	mq.workers["worker1"] = &MaintenanceWorker{
+		ID:            "worker1",
+		Status:        "active",
+		Capabilities:  []MaintenanceTaskType{"balance"},
+		MaxConcurrent: 1,
+		LastHeartbeat: time.Now(),
+	}
+
+	// Assign task
+	mq.GetNextTask("worker1", []MaintenanceTaskType{"balance"})
+
+	// Verify capacity reserved (9 left)
+	if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
+		t.Errorf("Expected capacity 9, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	}
+
+	// Make worker stale
+	mq.workers["worker1"].LastHeartbeat = time.Now().Add(-1 * time.Hour)
+
+	// Remove stale workers
+	mq.RemoveStaleWorkers(10 * time.Minute)
+
+	// Verify capacity released (back to 10)
+	if at.GetEffectiveAvailableCapacity("server1", 2) != 10 {
+		t.Errorf("Expected capacity 10 after removing stale worker, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	}
+}
+
+func TestMaintenanceManager_CancelTaskCapacityRelease(t *testing.T) {
+	// Setup Manager
+	config := DefaultMaintenanceConfig()
+	mm := NewMaintenanceManager(nil, config)
+	integration := mm.scanner.integration
+	mq := mm.queue
+	at := integration.GetActiveTopology()
+
+	topologyInfo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "server1",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd1": {DiskId: 1, VolumeCount: 1, MaxVolumeCount: 10},
+									"hdd2": {DiskId: 2, VolumeCount: 0, MaxVolumeCount: 10},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	at.UpdateTopology(topologyInfo)
+
+	taskID := "cancel_test_123"
+	// Note: AddPendingTask reserves capacity
+	at.AddPendingTask(topology.TaskSpec{
+		TaskID:       taskID,
+		TaskType:     topology.TaskTypeBalance,
+		VolumeID:     100,
+		VolumeSize:   1024,
+		Sources:      []topology.TaskSourceSpec{{ServerID: "server1", DiskID: 1}},
+		Destinations: []topology.TaskDestinationSpec{{ServerID: "server1", DiskID: 2}},
+	})
+
+	mq.AddTask(&MaintenanceTask{
+		ID:       taskID,
+		Type:     "balance",
+		VolumeID: 100,
+		Server:   "server1",
+		TypedParams: &worker_pb.TaskParams{
+			TaskId:  taskID,
+			Targets: []*worker_pb.TaskTarget{{Node: "server1", DiskId: 2}},
+		},
+	})
+
+	// Verify capacity reserved (9 left)
+	if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
+		t.Errorf("Expected capacity 9, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	}
+
+	// Cancel task
+	err := mm.CancelTask(taskID)
+	if err != nil {
+		t.Fatalf("Failed to cancel task: %v", err)
+	}
+
+	// Verify capacity released (back to 10)
+	if at.GetEffectiveAvailableCapacity("server1", 2) != 10 {
+		t.Errorf("Expected capacity 10 after cancelling task, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	}
+}
+
+type MockPersistence struct {
+	tasks []*MaintenanceTask
+}
+
+func (m *MockPersistence) SaveTaskState(task *MaintenanceTask) error                { return nil }
+func (m *MockPersistence) LoadTaskState(taskID string) (*MaintenanceTask, error)    { return nil, nil }
+func (m *MockPersistence) LoadAllTaskStates() ([]*MaintenanceTask, error)           { return m.tasks, nil }
+func (m *MockPersistence) DeleteTaskState(taskID string) error                      { return nil }
+func (m *MockPersistence) CleanupCompletedTasks() error                             { return nil }
+func (m *MockPersistence) SaveTaskPolicy(taskType string, policy *TaskPolicy) error { return nil }
+
+func TestMaintenanceQueue_LoadTasksCapacitySync(t *testing.T) {
+	// Setup
+	policy := &MaintenancePolicy{
+		TaskPolicies: map[string]*worker_pb.TaskPolicy{
+			"balance": {MaxConcurrent: 1},
+		},
+	}
+	mq := NewMaintenanceQueue(policy)
+	integration := NewMaintenanceIntegration(mq, policy)
+	mq.SetIntegration(integration)
+	at := integration.GetActiveTopology()
+
+	topologyInfo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "server1",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd1": {DiskId: 1, VolumeCount: 1, MaxVolumeCount: 10},
+									"hdd2": {DiskId: 2, VolumeCount: 0, MaxVolumeCount: 10},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	at.UpdateTopology(topologyInfo)
+
+	// Setup mock persistence with a pending task
+	taskID := "load_test_123"
+	mockTask := &MaintenanceTask{
+		ID:     taskID,
+		Type:   "balance",
+		Status: TaskStatusPending,
+		TypedParams: &worker_pb.TaskParams{
+			TaskId:  taskID,
+			Sources: []*worker_pb.TaskSource{{Node: "server1", DiskId: 1}},
+			Targets: []*worker_pb.TaskTarget{{Node: "server1", DiskId: 2}},
+		},
+	}
+	mq.SetPersistence(&MockPersistence{tasks: []*MaintenanceTask{mockTask}})
+
+	// Load tasks
+	err := mq.LoadTasksFromPersistence()
+	if err != nil {
+		t.Fatalf("Failed to load tasks: %v", err)
+	}
+
+	// Verify capacity is reserved in ActiveTopology after loading (9 left)
+	if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
+		t.Errorf("Expected capacity 9 after loading tasks, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	}
+}
+
+func TestMaintenanceQueue_RetryCapacitySync(t *testing.T) {
+	// Setup
+	policy := &MaintenancePolicy{
+		TaskPolicies: map[string]*worker_pb.TaskPolicy{
+			"balance": {MaxConcurrent: 1},
+		},
+	}
+	mq := NewMaintenanceQueue(policy)
+	integration := NewMaintenanceIntegration(mq, policy)
+	mq.SetIntegration(integration)
+	at := integration.GetActiveTopology()
+
+	topologyInfo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "server1",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd1": {DiskId: 1, VolumeCount: 1, MaxVolumeCount: 10},
+									"hdd2": {DiskId: 2, VolumeCount: 0, MaxVolumeCount: 10},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	at.UpdateTopology(topologyInfo)
+
+	taskID := "retry_test_123"
+	// 1. Add task
+	at.AddPendingTask(topology.TaskSpec{
+		TaskID:       taskID,
+		TaskType:     topology.TaskTypeBalance,
+		VolumeID:     100,
+		VolumeSize:   1024,
+		Sources:      []topology.TaskSourceSpec{{ServerID: "server1", DiskID: 1}},
+		Destinations: []topology.TaskDestinationSpec{{ServerID: "server1", DiskID: 2}},
+	})
+
+	mq.AddTask(&MaintenanceTask{
+		ID:         taskID,
+		Type:       "balance",
+		VolumeID:   100,
+		Server:     "server1",
+		MaxRetries: 3,
+		TypedParams: &worker_pb.TaskParams{
+			TaskId:  taskID,
+			Sources: []*worker_pb.TaskSource{{Node: "server1", DiskId: 1}},
+			Targets: []*worker_pb.TaskTarget{{Node: "server1", DiskId: 2}},
+		},
+	})
+
+	mq.workers["worker1"] = &MaintenanceWorker{
+		ID:            "worker1",
+		Status:        "active",
+		Capabilities:  []MaintenanceTaskType{"balance"},
+		MaxConcurrent: 1,
+		LastHeartbeat: time.Now(),
+	}
+
+	// 2. Assign task
+	mq.GetNextTask("worker1", []MaintenanceTaskType{"balance"})
+
+	// Verify capacity reserved (9 left)
+	if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
+		t.Errorf("Initial assignment: Expected capacity 9, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	}
+
+	// 3. Complete with error (trigger retry)
+	mq.CompleteTask(taskID, "simulated failure")
+
+	// 4. Verify state after failure
+	task := mq.tasks[taskID]
+	if task.Status != TaskStatusPending {
+		t.Errorf("Expected status pending for retry, got %v", task.Status)
+	}
+	if task.RetryCount != 1 {
+		t.Errorf("Expected retry count 1, got %d", task.RetryCount)
+	}
+
+	// 5. Verify capacity in ActiveTopology
+	// It should first release (back to 10) and then re-reserve (SyncTask) because it's pending again.
+	// So it should still be 9.
+	if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
+		t.Errorf("After retry sync: Expected capacity 9, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	}
+}
+
+func TestMaintenanceQueue_AssignTaskRollback(t *testing.T) {
+	// Setup Policy
+	policy := &MaintenancePolicy{
+		TaskPolicies: map[string]*worker_pb.TaskPolicy{
+			"balance": {MaxConcurrent: 1},
+		},
+		GlobalMaxConcurrent: 10,
+	}
+
+	// Setup Queue and Integration
+	mq := NewMaintenanceQueue(policy)
+	integration := NewMaintenanceIntegration(mq, policy)
+	mq.SetIntegration(integration)
+
+	// Get Topology
+	at := integration.GetActiveTopology()
+	topologyInfo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "server1",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {
+										DiskId:         1,
+										VolumeCount:    1,
+										MaxVolumeCount: 1, // Only 1 slot
+										VolumeInfos: []*master_pb.VolumeInformationMessage{
+											{Id: 100, Collection: "col1"},
+										},
+									},
+									"hdd2": {
+										DiskId:         2,
+										VolumeCount:    0,
+										MaxVolumeCount: 0, // NO CAPACITY for target
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	at.UpdateTopology(topologyInfo)
+
+	taskID := "rollback_test_123"
+
+	// 1. Add task to MaintenanceQueue ONLY
+	// It's not in ActiveTopology, so AssignTask will fail with "pending task not found"
+	mq.AddTask(&MaintenanceTask{
+		ID:         taskID,
+		Type:       MaintenanceTaskType("balance"),
+		VolumeID:   100,
+		Server:     "server1",
+		Collection: "col1",
+		TypedParams: &worker_pb.TaskParams{
+			TaskId: taskID,
+			Targets: []*worker_pb.TaskTarget{
+				{Node: "server1", DiskId: 2},
+			},
+		},
+	})
+
+	// 2. Setup worker
+	mq.workers["worker1"] = &MaintenanceWorker{
+		ID:            "worker1",
+		Status:        "active",
+		Capabilities:  []MaintenanceTaskType{"balance"},
+		MaxConcurrent: 10,
+	}
+
+	// 3. Try to get next task
+	taskFound := mq.GetNextTask("worker1", []MaintenanceTaskType{"balance"})
+
+	// 4. Verify GetNextTask returned nil due to ActiveTopology.AssignTask failure
+	if taskFound != nil {
+		t.Errorf("Expected GetNextTask to return nil, got task %s", taskFound.ID)
+	}
+
+	// 5. Verify the task in MaintenanceQueue is rolled back to pending
+	mq.mutex.RLock()
+	task, exists := mq.tasks[taskID]
+	mq.mutex.RUnlock()
+
+	if !exists {
+		t.Fatalf("Task %s should still exist in MaintenanceQueue", taskID)
+	}
+	if task.Status != TaskStatusPending {
+		t.Errorf("Expected task status %v, got %v", TaskStatusPending, task.Status)
+	}
+	if task.WorkerID != "" {
+		t.Errorf("Expected task WorkerID to be empty, got %s", task.WorkerID)
+	}
+	if len(task.AssignmentHistory) != 0 {
+		t.Errorf("Expected assignment history to be empty, got %d records", len(task.AssignmentHistory))
+	}
+
+	// 6. Verify the task is still in pendingTasks slice
+	mq.mutex.RLock()
+	foundInPending := false
+	for _, pt := range mq.pendingTasks {
+		if pt.ID == taskID {
+			foundInPending = true
+			break
+		}
+	}
+	mq.mutex.RUnlock()
+
+	if !foundInPending {
+		t.Errorf("Task %s should still be in pendingTasks slice", taskID)
+	}
+}

weed/admin/maintenance/maintenance_types.go

@@ -351,6 +351,7 @@ type MaintenanceScanner struct {
 
 // TaskDetectionResult represents the result of scanning for maintenance needs
 type TaskDetectionResult struct {
+	TaskID      string                  `json:"task_id"`
 	TaskType    MaintenanceTaskType     `json:"task_type"`
 	VolumeID    uint32                  `json:"volume_id,omitempty"`
 	Server      string                  `json:"server,omitempty"`

weed/admin/topology/task_management.go

@@ -66,11 +66,17 @@ func (at *ActiveTopology) CompleteTask(taskID string) error {
 
 	task, exists := at.assignedTasks[taskID]
 	if !exists {
-		return fmt.Errorf("assigned task %s not found", taskID)
+		// If not in assigned tasks, check pending tasks
+		if task, exists = at.pendingTasks[taskID]; exists {
+			delete(at.pendingTasks, taskID)
+		} else {
+			return fmt.Errorf("task %s not found in assigned or pending tasks", taskID)
+		}
+	} else {
+		delete(at.assignedTasks, taskID)
 	}
 
 	// Release reserved capacity by moving task to completed state
-	delete(at.assignedTasks, taskID)
 	task.Status = TaskStatusCompleted
 	task.CompletedAt = time.Now()
 	at.recentTasks[taskID] = task

weed/admin/view/app/admin_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_brokers_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_collections_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_ec_shards_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_ec_volumes_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_filers_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_masters_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_volume_servers_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_volumes_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/collection_details_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/ec_volume_details_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/file_browser_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/iceberg_catalog_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/iceberg_namespaces_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/iceberg_tables_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/maintenance_config_schema_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/maintenance_config_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/maintenance_queue.templ

@@ -299,16 +299,6 @@ templ MaintenanceQueue(data *maintenance.MaintenanceQueueData) {
     </div>
 
     <script>
-        // Debug output to browser console
-        console.log("DEBUG: Maintenance Queue Template loaded");
-        
-        // Auto-refresh every 10 seconds
-        setInterval(function() {
-            if (!document.hidden) {
-                window.location.reload();
-            }
-        }, 10000);
-
         window.triggerScan = function() {
             console.log("triggerScan called");
             fetch('/api/maintenance/scan', {

weed/admin/view/app/maintenance_queue_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.
@@ -610,7 +610,7 @@ func MaintenanceQueue(data *maintenance.MaintenanceQueueData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</div></div></div></div></div><script>\n        // Debug output to browser console\n        console.log(\"DEBUG: Maintenance Queue Template loaded\");\n        \n        // Auto-refresh every 10 seconds\n        setInterval(function() {\n            if (!document.hidden) {\n                window.location.reload();\n            }\n        }, 10000);\n\n        window.triggerScan = function() {\n            console.log(\"triggerScan called\");\n            fetch('/api/maintenance/scan', {\n                method: 'POST',\n                headers: {\n                    'Content-Type': 'application/json',\n                }\n            })\n            .then(response => response.json())\n            .then(data => {\n                if (data.success) {\n                    showToast('Success', 'Maintenance scan triggered successfully', 'success');\n                    setTimeout(() => window.location.reload(), 2000);\n                } else {\n                    showToast('Error', 'Failed to trigger scan: ' + (data.error || 'Unknown error'), 'danger');\n                }\n            })\n            .catch(error => {\n                showToast('Error', 'Error: ' + error.message, 'danger');\n            });\n        };\n\n        window.refreshPage = function() {\n            console.log(\"refreshPage called\");\n            window.location.reload();\n        };\n\n        window.navigateToTask = function(element) {\n            const taskId = element.getAttribute('data-task-id');\n            if (taskId) {\n                window.location.href = '/maintenance/tasks/' + taskId;\n            }\n        };\n    </script>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</div></div></div></div></div><script>\n        window.triggerScan = function() {\n            console.log(\"triggerScan called\");\n            fetch('/api/maintenance/scan', {\n                method: 'POST',\n                headers: {\n                    'Content-Type': 'application/json',\n                }\n            })\n            .then(response => response.json())\n            .then(data => {\n                if (data.success) {\n                    showToast('Success', 'Maintenance scan triggered successfully', 'success');\n                    setTimeout(() => window.location.reload(), 2000);\n                } else {\n                    showToast('Error', 'Failed to trigger scan: ' + (data.error || 'Unknown error'), 'danger');\n                }\n            })\n            .catch(error => {\n                showToast('Error', 'Error: ' + error.message, 'danger');\n            });\n        };\n\n        window.refreshPage = function() {\n            console.log(\"refreshPage called\");\n            window.location.reload();\n        };\n\n        window.navigateToTask = function(element) {\n            const taskId = element.getAttribute('data-task-id');\n            if (taskId) {\n                window.location.href = '/maintenance/tasks/' + taskId;\n            }\n        };\n    </script>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -809,7 +809,7 @@ func ProgressBar(progress float64, status maintenance.MaintenanceTaskStatus) tem
 			var templ_7745c5c3_Var35 string
 			templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.SanitizeStyleAttributeValues(fmt.Sprintf("width: %.1f%%", progress))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/maintenance_queue.templ`, Line: 390, Col: 102}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/maintenance_queue.templ`, Line: 380, Col: 102}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35))
 			if templ_7745c5c3_Err != nil {
@@ -822,7 +822,7 @@ func ProgressBar(progress float64, status maintenance.MaintenanceTaskStatus) tem
 			var templ_7745c5c3_Var36 string
 			templ_7745c5c3_Var36, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1f%%", progress))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/maintenance_queue.templ`, Line: 393, Col: 66}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/maintenance_queue.templ`, Line: 383, Col: 66}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var36))
 			if templ_7745c5c3_Err != nil {

weed/admin/view/app/maintenance_workers_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/object_store_users_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/policies_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/s3_buckets_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/s3tables_buckets_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/s3tables_namespaces_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/s3tables_tables_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/service_accounts_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/subscribers_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/task_config_schema.templ

@@ -430,7 +430,7 @@ func getTaskConfigStringField(config interface{}, fieldName string) string {
 
 func getTaskNumberStep(field *config.Field) string {
     if field.Type == config.FieldTypeFloat {
-        return "0.01"
+        return "any"
     }
     return "1"
 }

weed/admin/view/app/task_config_schema_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.
@@ -891,7 +891,7 @@ func getTaskConfigStringField(config interface{}, fieldName string) string {
 
 func getTaskNumberStep(field *config.Field) string {
 	if field.Type == config.FieldTypeFloat {
-		return "0.01"
+		return "any"
 	}
 	return "1"
 }

weed/admin/view/app/task_config_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/task_config_templ_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/task_detail.templ

@@ -942,10 +942,48 @@ templ TaskDetail(data *maintenance.TaskDetailData) {
             return;
         }
         
+        
         // Format and display logs with structured fields
         let logText = '';
+        
+        // Helper function to format timestamps robustly
+        function formatTimestamp(timestamp) {
+            if (!timestamp) {
+                return 'N/A';
+            }
+            
+            let date;
+            
+            // Check if timestamp is a numeric string (e.g., "1738652668")
+            if (typeof timestamp === 'string' && /^\d+$/.test(timestamp)) {
+                const numericTimestamp = parseInt(timestamp, 10);
+                // Treat values > 10^10 as milliseconds, otherwise as seconds
+                date = numericTimestamp > 10000000000 
+                    ? new Date(numericTimestamp) 
+                    : new Date(numericTimestamp * 1000);
+            } else if (typeof timestamp === 'string') {
+                // ISO date string
+                date = new Date(timestamp);
+            } else if (typeof timestamp === 'number') {
+                // Numeric timestamp (seconds or milliseconds)
+                date = timestamp > 10000000000 
+                    ? new Date(timestamp) 
+                    : new Date(timestamp * 1000);
+            } else {
+                return 'N/A';
+            }
+            
+            // Validate the date
+            if (isNaN(date.getTime())) {
+                return 'N/A';
+            }
+            
+            return date.toISOString();
+        }
+        
         logs.forEach(entry => {
-            const timestamp = entry.timestamp ? new Date(entry.timestamp * 1000).toISOString() : 'N/A';
+            const timestamp = formatTimestamp(entry.timestamp);
+
             const level = entry.level || 'INFO';
             const message = entry.message || '';
             
@@ -1011,7 +1049,12 @@ templ TaskDetail(data *maintenance.TaskDetailData) {
             let logContent = '';
             if (data.logs && data.logs.length > 0) {
                 data.logs.forEach(entry => {
-                    const timestamp = entry.timestamp ? new Date(entry.timestamp * 1000).toISOString() : 'N/A';
+                    let timestamp;
+                    if (typeof entry.timestamp === 'string') {
+                        timestamp = new Date(entry.timestamp).toISOString();
+                    } else {
+                        timestamp = entry.timestamp ? new Date(entry.timestamp * 1000).toISOString() : 'N/A';
+                    }
                     const level = entry.level || 'INFO';
                     const message = entry.message || '';
                     

weed/admin/view/app/topic_details_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/topics_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/volume_details_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/components/config_sections_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package components
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/components/form_fields_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package components
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/layout/layout_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.960
+// templ: version: v0.3.977
 package layout
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/command/admin.go

@@ -230,7 +230,20 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 
 	// Create router
 	r := gin.New()
-	r.Use(gin.Logger(), gin.Recovery())
+	r.Use(gin.LoggerWithFormatter(func(param gin.LogFormatterParams) string {
+		if param.StatusCode == 200 {
+			return ""
+		}
+		return fmt.Sprintf("[GIN] %v | %3d | %13v | %15s | %-7s %s\n%s",
+			param.TimeStamp.Format("2006/01/02 - 15:04:05"),
+			param.StatusCode,
+			param.Latency,
+			param.ClientIP,
+			param.Method,
+			param.Path,
+			param.ErrorMessage,
+		)
+	}), gin.Recovery())
 
 	// Create data directory first if specified (needed for session key storage)
 	var dataDir string

weed/command/filer.go

@@ -143,6 +143,8 @@ func init() {
 	filerS3Options.concurrentFileUploadLimit = cmdFiler.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
 	filerS3Options.enableIam = cmdFiler.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port")
 	filerS3Options.cipher = cmdFiler.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
+	filerS3Options.iamReadOnly = cmdFiler.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server")
+	filerS3Options.portIceberg = cmdFiler.Flag.Int("s3.port.iceberg", 8181, "Iceberg REST Catalog server listen port (0 to disable)")
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")

weed/command/mini.go

@@ -93,12 +93,13 @@ Example Usage:
 	weed mini -dir=/data -master.port=9444  # Custom master port
 
 After starting, you can access:
-- Master UI:    http://localhost:9333
-- Volume Server: http://localhost:9340
-- Filer UI:     http://localhost:8888
-- S3 Endpoint:  http://localhost:8333
-- WebDAV:       http://localhost:7333
-- Admin UI:     http://localhost:23646
+- Master UI:       http://localhost:9333
+- Volume Server:   http://localhost:9340
+- Filer UI:        http://localhost:8888
+- S3 Endpoint:     http://localhost:8333
+- Iceberg Catalog: http://localhost:8181
+- WebDAV:          http://localhost:7333
+- Admin UI:        http://localhost:23646
 
 S3 Access:
 The S3 endpoint is available at http://localhost:8333. For client
@@ -887,9 +888,12 @@ func startMiniServices(miniWhiteList []string, allServicesReady chan struct{}) {
 		}, *miniWebDavOptions.port)
 	}
 
-	// Wait for both S3 and WebDAV to be ready
+	// Wait for services to be ready
 	if *miniEnableS3 {
 		waitForServiceReady("S3", *miniS3Options.port, bindIp)
+		if miniS3Options.portIceberg != nil && *miniS3Options.portIceberg > 0 {
+			waitForServiceReady("Iceberg", *miniS3Options.portIceberg, bindIp)
+		}
 	}
 	if *miniEnableWebDAV {
 		waitForServiceReady("WebDAV", *miniWebDavOptions.port, bindIp)
@@ -908,6 +912,7 @@ func startMiniService(name string, fn func(), port int) {
 // waitForServiceReady pings the service HTTP endpoint to check if it's ready to accept connections
 func waitForServiceReady(name string, port int, bindIp string) {
 	address := fmt.Sprintf("http://%s:%d", bindIp, port)
+	healthAddr := getHealthCheckAddr(address)
 	maxAttempts := 30 // 30 * 200ms = 6 seconds max wait
 	attempt := 0
 	client := &http.Client{
@@ -915,7 +920,7 @@ func waitForServiceReady(name string, port int, bindIp string) {
 	}
 
 	for attempt < maxAttempts {
-		resp, err := client.Get(address)
+		resp, err := client.Get(healthAddr)
 		if err == nil {
 			resp.Body.Close()
 			glog.Infof("%s service is ready at %s", name, address)
@@ -958,8 +963,8 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
 	// Determine bind IP for health checks
 	bindIp := getBindIp()
 
-	// Prepare master address
-	masterAddr := fmt.Sprintf("%s:%d", *miniIp, *miniMasterOptions.port)
+	// Prepare master address with gRPC port
+	masterAddr := string(pb.NewServerAddress(*miniIp, *miniMasterOptions.port, *miniMasterOptions.portGrpc))
 
 	// Set admin options
 	*miniAdminOptions.master = masterAddr
@@ -1021,7 +1026,7 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
 
 // waitForAdminServerReady pings the admin server HTTP endpoint to check if it's ready
 func waitForAdminServerReady(adminAddr string) error {
-	healthAddr := fmt.Sprintf("%s/health", adminAddr)
+	healthAddr := getHealthCheckAddr(fmt.Sprintf("%s/health", adminAddr))
 	maxAttempts := 60 // 60 * 500ms = 30 seconds max wait
 	attempt := 0
 	client := &http.Client{
@@ -1032,7 +1037,7 @@ func waitForAdminServerReady(adminAddr string) error {
 		resp, err := client.Get(healthAddr)
 		if err == nil {
 			resp.Body.Close()
-			glog.V(1).Infof("Admin server is ready at %s", adminAddr)
+			glog.Infof("Admin server is ready at %s", adminAddr)
 			return nil
 		}
 		attempt++
@@ -1041,6 +1046,12 @@ func waitForAdminServerReady(adminAddr string) error {
 
 	return fmt.Errorf("admin server did not become ready at %s after %d attempts", adminAddr, maxAttempts)
 }
+func getHealthCheckAddr(addr string) string {
+	if strings.Contains(addr, "://0.0.0.0:") {
+		return strings.Replace(addr, "://0.0.0.0:", "://127.0.0.1:", 1)
+	}
+	return addr
+}
 
 // waitForWorkerReady polls the worker's gRPC port to ensure the worker has fully initialized
 func waitForWorkerReady(workerGrpcAddr string) {
@@ -1170,20 +1181,23 @@ func printWelcomeMessage() {
 	sb.WriteString("║                      SeaweedFS Mini - All-in-One Mode                         ║\n")
 	sb.WriteString("╚═══════════════════════════════════════════════════════════════════════════════╝\n\n")
 	sb.WriteString("  All enabled components are running and ready to use:\n\n")
-	fmt.Fprintf(&sb, "    Master UI:      http://%s:%d\n", *miniIp, *miniMasterOptions.port)
-	fmt.Fprintf(&sb, "    Filer UI:       http://%s:%d\n", *miniIp, *miniFilerOptions.port)
+	fmt.Fprintf(&sb, "    Master UI:       http://%s:%d\n", *miniIp, *miniMasterOptions.port)
+	fmt.Fprintf(&sb, "    Filer UI:        http://%s:%d\n", *miniIp, *miniFilerOptions.port)
 	if *miniEnableS3 {
-		fmt.Fprintf(&sb, "    S3 Endpoint:    http://%s:%d\n", *miniIp, *miniS3Options.port)
+		fmt.Fprintf(&sb, "    S3 Endpoint:     http://%s:%d\n", *miniIp, *miniS3Options.port)
+		if miniS3Options.portIceberg != nil && *miniS3Options.portIceberg > 0 {
+			fmt.Fprintf(&sb, "    Iceberg Catalog: http://%s:%d\n", *miniIp, *miniS3Options.portIceberg)
+		}
 	}
 
 	if *miniEnableWebDAV {
-		fmt.Fprintf(&sb, "    WebDAV:         http://%s:%d\n", *miniIp, *miniWebDavOptions.port)
+		fmt.Fprintf(&sb, "    WebDAV:          http://%s:%d\n", *miniIp, *miniWebDavOptions.port)
 	}
 	if *miniEnableAdminUI {
-		fmt.Fprintf(&sb, "    Admin UI:       http://%s:%d\n", *miniIp, *miniAdminOptions.port)
+		fmt.Fprintf(&sb, "    Admin UI:        http://%s:%d\n", *miniIp, *miniAdminOptions.port)
 	}
 
-	fmt.Fprintf(&sb, "    Volume Server:  http://%s:%d\n\n", *miniIp, *miniOptions.v.port)
+	fmt.Fprintf(&sb, "    Volume Server:   http://%s:%d\n\n", *miniIp, *miniOptions.v.port)
 
 	sb.WriteString("  Optimized Settings:\n")
 	fmt.Fprintf(&sb, "    • Volume size limit: %dMB\n", *miniMasterOptions.volumeSizeLimitMB)

weed/command/mount.go

@@ -24,6 +24,7 @@ type MountOptions struct {
 	cacheSizeMBForRead *int64
 	dataCenter         *string
 	allowOthers        *bool
+	defaultPermissions *bool
 	umaskString        *string
 	nonempty           *bool
 	volumeServerAccess *string
@@ -86,6 +87,7 @@ func init() {
 	mountOptions.cacheMetaTtlSec = cmdMount.Flag.Int("cacheMetaTtlSec", 60, "metadata cache validity seconds")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
+	mountOptions.defaultPermissions = cmdMount.Flag.Bool("defaultPermissions", true, "enforce permissions by the operating system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
 	mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory")
 	mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")

weed/command/mount_std.go

@@ -188,6 +188,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		//SyncRead:                 false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
 		EnableAcl: true,
 	}
+	if *option.defaultPermissions {
+		fuseMountOptions.Options = append(fuseMountOptions.Options, "default_permissions")
+	}
 	if *option.nonempty {
 		fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
 	}
@@ -216,8 +219,12 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		fuseMountOptions.Options = append(fuseMountOptions.Options, fmt.Sprintf("iosize=%d", ioSizeMB*1024*1024))
 	}
 
-	fuseMountOptions.EnableWriteback = *option.writebackCache
-	fuseMountOptions.EnableAsyncDio = *option.asyncDio
+  if option.writebackCache != nil {
+	  fuseMountOptions.EnableWriteback = *option.writebackCache
+  }
+  if option.asyncDio != nil {
+	  fuseMountOptions.EnableAsyncDio = *option.asyncDio
+  }
 	if option.cacheSymlink != nil && *option.cacheSymlink {
 		fuseMountOptions.EnableSymlinkCaching = true
 	}

weed/command/server.go

@@ -152,6 +152,9 @@ func init() {
 	serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
 	serverOptions.v.inflightDownloadDataTimeout = cmdServer.Flag.Duration("volume.inflightDownloadDataTimeout", 60*time.Second, "inflight download data wait timeout of volume servers")
 
+	serverOptions.v.udsListen = cmdServer.Flag.String("volume.uds.listen", "", "Unix domain socket path for RDMA sidecar locate API (e.g., /tmp/sra-volume.sock)")
+	serverOptions.v.udsTransport = cmdServer.Flag.String("volume.uds.transport", "", "Unix domain socket path for outbound RDMA replication (e.g., /tmp/sra-transport.sock)")
+
 	serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
 	serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally")
 

weed/command/volume.go

@@ -74,6 +74,8 @@ type VolumeServerOptions struct {
 	ldbTimeout                  *int64
 	debug                       *bool
 	debugPort                   *int
+	udsListen                   *string // UDS socket path for RDMA sidecar integration
+	udsTransport                *string // UDS socket path for outbound RDMA replication
 }
 
 func init() {
@@ -114,6 +116,8 @@ func init() {
 	v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
 	v.debug = cmdVolume.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
 	v.debugPort = cmdVolume.Flag.Int("debug.port", 6060, "http port for debugging")
+	v.udsListen = cmdVolume.Flag.String("uds.listen", "", "Unix domain socket path for RDMA sidecar locate API (e.g., /tmp/sra-volume.sock)")
+	v.udsTransport = cmdVolume.Flag.String("uds.transport", "", "Unix domain socket path for outbound RDMA replication (e.g., /tmp/sra-transport.sock)")
 }
 
 var cmdVolume = &Command{
@@ -289,6 +293,22 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	// starting grpc server
 	grpcS := v.startGrpcService(volumeServer)
 
+	// starting UDS server for RDMA sidecar integration
+	var udsServer *weed_server.UdsServer
+	if *v.udsListen != "" {
+		var err error
+		udsServer, err = weed_server.NewUdsServer(volumeServer, *v.udsListen)
+		if err != nil {
+			glog.Fatalf("failed to start UDS server: %v", err)
+		}
+		udsServer.Start()
+	}
+
+	// set up outbound RDMA replication transport
+	if *v.udsTransport != "" {
+		volumeServer.SetSraTransport(storage.NewSraTransport(*v.udsTransport))
+	}
+
 	// starting public http server
 	var publicHttpDown httpdown.Server
 	if v.isSeparatedPublicPort() {
@@ -315,7 +335,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 			time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)
 		}
 
-		shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
+		shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer, udsServer)
 		stopChan <- true
 	})
 
@@ -324,7 +344,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		select {
 		case <-stopChan:
 		case <-ctx.Done():
-			shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
+			shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer, udsServer)
 		}
 	} else {
 		select {
@@ -334,7 +354,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 }
 
-func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {
+func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer, udsServer *weed_server.UdsServer) {
 
 	// firstly, stop the public http service to prevent from receiving new user request
 	if nil != publicHttpDown {
@@ -352,6 +372,11 @@ func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server,
 	glog.V(0).Infof("graceful stop gRPC ...")
 	grpcS.GracefulStop()
 
+	if udsServer != nil {
+		glog.V(0).Infof("stop UDS server ...")
+		udsServer.Stop()
+	}
+
 	volumeServer.Shutdown()
 
 	pprof.StopCPUProfile()

weed/filer/filechunks.go

@@ -59,7 +59,8 @@ func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
 	for _, c := range chunks {
 		md5Digests = append(md5Digests, util.Base64Md5ToBytes(c.ETag))
 	}
-	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5Digests, nil)), len(chunks))
+	finalETag := fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5Digests, nil)), len(chunks))
+	return finalETag
 }
 
 func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {

weed/operation/upload_chunked.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/md5"
+	"encoding/base64"
 	"fmt"
 	"hash"
 	"io"
@@ -171,6 +172,10 @@ uploadLoop:
 				jwt = assignResult.Auth
 			}
 
+			// Calculate MD5 for the chunk
+			chunkMd5 := md5.Sum(buf.Bytes())
+			chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
+
 			uploadOption := &UploadOption{
 				UploadUrl:         uploadUrl,
 				Cipher:            opt.Cipher,
@@ -178,6 +183,7 @@ uploadLoop:
 				MimeType:          opt.MimeType,
 				PairMap:           nil,
 				Jwt:               jwt,
+				Md5:               chunkMd5B64,
 			}
 
 			var uploadResult *UploadResult
@@ -225,7 +231,6 @@ uploadLoop:
 			}
 			fileChunksLock.Lock()
 			fileChunks = append(fileChunks, chunk)
-			glog.V(4).Infof("uploaded chunk %d to %s [%d,%d)", len(fileChunks), chunk.FileId, offset, offset+int64(chunk.Size))
 			fileChunksLock.Unlock()
 
 		}(chunkOffset, bytesBuffer)

weed/pb/server_address.go

@@ -15,7 +15,7 @@ type ServerAddresses string
 type ServerSrvAddress string
 
 func NewServerAddress(host string, port int, grpcPort int) ServerAddress {
-	if grpcPort == 0 || grpcPort == port+10000 {
+	if grpcPort == 0 {
 		return ServerAddress(util.JoinHostPort(host, port))
 	}
 	return ServerAddress(util.JoinHostPort(host, port) + "." + strconv.Itoa(grpcPort))
@@ -25,10 +25,6 @@ func NewServerAddressWithGrpcPort(address string, grpcPort int) ServerAddress {
 	if grpcPort == 0 {
 		return ServerAddress(address)
 	}
-	_, port, _ := hostAndPort(address)
-	if uint64(grpcPort) == port+10000 {
-		return ServerAddress(address)
-	}
 	return ServerAddress(address + "." + strconv.Itoa(grpcPort))
 }
 

weed/pb/volume_server.proto

@@ -9,8 +9,10 @@ import "remote.proto";
 
 // Persistent state for volume servers.
 message VolumeServerState {
-    // Whether the server is in maintenance (i.e. read-only) mode.
+    // whether the server is in maintenance (i.e. read-only) mode.
     bool maintenance = 1;
+    // incremental version counter
+    uint32 version = 2;
 }
 
 //////////////////////////////////////////////////
@@ -643,6 +645,7 @@ enum VolumeScrubMode {
     UNKNOWN = 0;
     INDEX = 1;
     FULL = 2;
+    LOCAL = 3;
 }
 
 message ScrubVolumeRequest {

weed/pb/volume_server_pb/volume_server.pb.go

@@ -28,6 +28,7 @@ const (
 	VolumeScrubMode_UNKNOWN VolumeScrubMode = 0
 	VolumeScrubMode_INDEX   VolumeScrubMode = 1
 	VolumeScrubMode_FULL    VolumeScrubMode = 2
+	VolumeScrubMode_LOCAL   VolumeScrubMode = 3
 )
 
 // Enum value maps for VolumeScrubMode.
@@ -36,11 +37,13 @@ var (
 		0: "UNKNOWN",
 		1: "INDEX",
 		2: "FULL",
+		3: "LOCAL",
 	}
 	VolumeScrubMode_value = map[string]int32{
 		"UNKNOWN": 0,
 		"INDEX":   1,
 		"FULL":    2,
+		"LOCAL":   3,
 	}
 )
 
@@ -74,8 +77,10 @@ func (VolumeScrubMode) EnumDescriptor() ([]byte, []int) {
 // Persistent state for volume servers.
 type VolumeServerState struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
-	// Whether the server is in maintenance (i.e. read-only) mode.
-	Maintenance   bool `protobuf:"varint,1,opt,name=maintenance,proto3" json:"maintenance,omitempty"`
+	// whether the server is in maintenance (i.e. read-only) mode.
+	Maintenance bool `protobuf:"varint,1,opt,name=maintenance,proto3" json:"maintenance,omitempty"`
+	// incremental version counter
+	Version       uint32 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -117,6 +122,13 @@ func (x *VolumeServerState) GetMaintenance() bool {
 	return false
 }
 
+func (x *VolumeServerState) GetVersion() uint32 {
+	if x != nil {
+		return x.Version
+	}
+	return 0
+}
+
 type BatchDeleteRequest struct {
 	state           protoimpl.MessageState `protogen:"open.v1"`
 	FileIds         []string               `protobuf:"bytes,1,rep,name=file_ids,json=fileIds,proto3" json:"file_ids,omitempty"`
@@ -1855,7 +1867,7 @@ func (x *GetStateResponse) GetState() *VolumeServerState {
 
 type SetStateRequest struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
-	// SetState updates *all* volume server flags at once. Retrieve state with GetState(),
+	// SetState updates *all* volume server flags at once. Retrieve state/version with GetState(),
 	// modify individual flags as required, then call this RPC to update.
 	State         *VolumeServerState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"`
 	unknownFields protoimpl.UnknownFields
@@ -6687,9 +6699,10 @@ var File_volume_server_proto protoreflect.FileDescriptor
 
 const file_volume_server_proto_rawDesc = "" +
 	"\n" +
-	"\x13volume_server.proto\x12\x10volume_server_pb\x1a\fremote.proto\"5\n" +
+	"\x13volume_server.proto\x12\x10volume_server_pb\x1a\fremote.proto\"O\n" +
 	"\x11VolumeServerState\x12 \n" +
-	"\vmaintenance\x18\x01 \x01(\bR\vmaintenance\"[\n" +
+	"\vmaintenance\x18\x01 \x01(\bR\vmaintenance\x12\x18\n" +
+	"\aversion\x18\x02 \x01(\rR\aversion\"[\n" +
 	"\x12BatchDeleteRequest\x12\x19\n" +
 	"\bfile_ids\x18\x01 \x03(\tR\afileIds\x12*\n" +
 	"\x11skip_cookie_check\x18\x02 \x01(\bR\x0fskipCookieCheck\"O\n" +
@@ -7180,11 +7193,12 @@ const file_volume_server_proto_rawDesc = "" +
 	"\rstart_time_ns\x18\x01 \x01(\x03R\vstartTimeNs\x12$\n" +
 	"\x0eremote_time_ns\x18\x02 \x01(\x03R\fremoteTimeNs\x12 \n" +
 	"\fstop_time_ns\x18\x03 \x01(\x03R\n" +
-	"stopTimeNs*3\n" +
+	"stopTimeNs*>\n" +
 	"\x0fVolumeScrubMode\x12\v\n" +
 	"\aUNKNOWN\x10\x00\x12\t\n" +
 	"\x05INDEX\x10\x01\x12\b\n" +
-	"\x04FULL\x10\x022\xfb(\n" +
+	"\x04FULL\x10\x02\x12\t\n" +
+	"\x05LOCAL\x10\x032\xfb(\n" +
 	"\fVolumeServer\x12\\\n" +
 	"\vBatchDelete\x12$.volume_server_pb.BatchDeleteRequest\x1a%.volume_server_pb.BatchDeleteResponse\"\x00\x12n\n" +
 	"\x11VacuumVolumeCheck\x12*.volume_server_pb.VacuumVolumeCheckRequest\x1a+.volume_server_pb.VacuumVolumeCheckResponse\"\x00\x12v\n" +

weed/s3api/auth_credentials.go

@@ -538,17 +538,45 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
 	}
 
 	iam.m.Lock()
+	// Save existing environment-based identities before replacement
+	// This ensures AWS_ACCESS_KEY_ID credentials are preserved
+	envIdentities := make([]*Identity, 0)
+	for _, ident := range iam.identities {
+		if ident.IsStatic && strings.HasPrefix(ident.Name, "admin-") {
+			// This is an environment-based admin identity, preserve it
+			envIdentities = append(envIdentities, ident)
+		}
+	}
+
 	// atomically switch
 	iam.identities = identities
 	iam.identityAnonymous = identityAnonymous
 	iam.accounts = accounts
 	iam.emailAccount = emailAccount
-	iam.accessKeyIdent = accessKeyIdent
 	iam.nameToIdentity = nameToIdentity
+	iam.accessKeyIdent = accessKeyIdent
 	iam.policies = policies
+
+	// Re-add environment-based identities that were preserved
+	for _, envIdent := range envIdentities {
+		// Check if this identity already exists in the new config
+		exists := false
+		for _, ident := range iam.identities {
+			if ident.Name == envIdent.Name {
+				exists = true
+				break
+			}
+		}
+		if !exists {
+			iam.identities = append(iam.identities, envIdent)
+			iam.accessKeyIdent[envIdent.Credentials[0].AccessKey] = envIdent
+			iam.nameToIdentity[envIdent.Name] = envIdent
+		}
+	}
+
 	// Update authentication state based on whether identities exist
 	// Once enabled, keep it enabled (one-way toggle)
-	authJustEnabled := iam.updateAuthenticationState(len(identities))
+	authJustEnabled := iam.updateAuthenticationState(len(iam.identities))
 	iam.m.Unlock()
 
 	if authJustEnabled {
@@ -778,9 +806,10 @@ func (iam *IdentityAccessManagement) MergeS3ApiConfiguration(config *iam_pb.S3Ap
 	iam.identityAnonymous = identityAnonymous
 	iam.accounts = accounts
 	iam.emailAccount = emailAccount
-	iam.accessKeyIdent = accessKeyIdent
 	iam.nameToIdentity = nameToIdentity
+	iam.accessKeyIdent = accessKeyIdent
 	iam.policies = policies
+	iam.accessKeyIdent = accessKeyIdent
 	// Update authentication state based on whether identities exist
 	// Once enabled, keep it enabled (one-way toggle)
 	authJustEnabled := iam.updateAuthenticationState(len(identities))

weed/s3api/auth_signature_v4.go

@@ -22,6 +22,7 @@ import (
 	"crypto/hmac"
 	"crypto/sha256"
 	"crypto/subtle"
+	"encoding/base64"
 	"encoding/hex"
 	"io"
 	"net"
@@ -104,6 +105,24 @@ func getContentSha256Cksum(r *http.Request) string {
 	return emptySHA256
 }
 
+// normalizePayloadHash converts base64-encoded payload hash to hex format.
+// AWS SigV4 canonical requests always use hex-encoded SHA256.
+func normalizePayloadHash(payloadHashValue string) string {
+	// Special values and hex-encoded hashes don't need conversion
+	if payloadHashValue == emptySHA256 || payloadHashValue == unsignedPayload || 
+		payloadHashValue == streamingContentSHA256 || payloadHashValue == streamingContentSHA256Trailer ||
+		payloadHashValue == streamingUnsignedPayload || len(payloadHashValue) == 64 {
+		return payloadHashValue
+	}
+
+	// Try to decode as base64 and convert to hex
+	if decodedBytes, err := base64.StdEncoding.DecodeString(payloadHashValue); err == nil && len(decodedBytes) == 32 {
+		return hex.EncodeToString(decodedBytes)
+	}
+
+	return payloadHashValue
+}
+
 // signValues data type represents structured form of AWS Signature V4 header.
 type signValues struct {
 	Credential    credentialHeader
@@ -485,6 +504,10 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode)
 		}
 	}
 
+	// Normalize payload hash to hex format for canonical request
+	// AWS SigV4 canonical requests always use hex-encoded SHA256
+	normalizedPayload := normalizePayloadHash(hashedPayload)
+
 	return &v4AuthInfo{
 		Signature:     signV4Values.Signature,
 		AccessKey:     signV4Values.Credential.accessKey,
@@ -493,7 +516,7 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode)
 		Region:        signV4Values.Credential.scope.region,
 		Service:       signV4Values.Credential.scope.service,
 		Scope:         signV4Values.Credential.getScope(),
-		HashedPayload: hashedPayload,
+		HashedPayload: normalizedPayload,
 		IsPresigned:   false,
 	}, s3err.ErrNone
 }

weed/s3api/filer_multipart.go

@@ -2,6 +2,7 @@ package s3api
 
 import (
 	"cmp"
+	"crypto/md5"
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/hex"
@@ -206,7 +207,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 				return &CompleteMultipartUploadResult{
 					Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
 					Bucket:   input.Bucket,
-					ETag:     aws.String("\"" + filer.ETagChunks(entry.GetChunks()) + "\""),
+					ETag:     aws.String(getEtagFromEntry(entry)),
 					Key:      objectKey(input.Key),
 				}, s3err.ErrNone
 			}
@@ -301,10 +302,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 			return nil, s3err.ErrInvalidPart
 		}
 		found := false
+
 		if len(partEntriesByNumber) > 1 {
-			slices.SortFunc(partEntriesByNumber, func(a, b *filer_pb.Entry) int {
-				return cmp.Compare(b.Chunks[0].ModifiedTsNs, a.Chunks[0].ModifiedTsNs)
-			})
+			sortEntriesByLatestChunk(partEntriesByNumber)
 		}
 		for _, entry := range partEntriesByNumber {
 			if found {
@@ -357,6 +357,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 
 	entryName, dirName := s3a.getEntryNameAndDir(input)
 
+	// Precompute ETag once for consistency across all paths
+	multipartETag := calculateMultipartETag(partEntries, completedPartNumbers)
+	etagQuote := "\"" + multipartETag + "\""
+
 	// Check if versioning is configured for this bucket BEFORE creating any files
 	versioningState, vErr := s3a.getVersioningState(*input.Bucket)
 	if vErr == nil && versioningState == s3_constants.VersioningEnabled {
@@ -396,6 +400,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 				}
 			}
 
+			// Persist ETag to ensure subsequent HEAD/GET uses the same value
+			versionEntry.Extended[s3_constants.ExtETagKey] = []byte(multipartETag)
+
 			// Preserve ALL SSE metadata from the first part (if any)
 			// SSE metadata is stored in individual parts, not the upload directory
 			if len(completedPartNumbers) > 0 && len(partEntries[completedPartNumbers[0]]) > 0 {
@@ -418,14 +425,14 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 
 		// Construct entry with metadata for caching in .versions directory
 		// Reuse versionMtime to keep list vs. HEAD timestamps aligned
-		etag := "\"" + filer.ETagChunks(finalParts) + "\""
+		// multipartETag is precomputed
 		versionEntryForCache := &filer_pb.Entry{
 			Attributes: &filer_pb.FuseAttributes{
 				FileSize: uint64(offset),
 				Mtime:    versionMtime,
 			},
 			Extended: map[string][]byte{
-				s3_constants.ExtETagKey: []byte(etag),
+				s3_constants.ExtETagKey: []byte(multipartETag),
 			},
 		}
 		if amzAccountId != "" {
@@ -440,13 +447,12 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 			return nil, s3err.ErrInternalError
 		}
 
-		// For versioned buckets, don't create a main object file - all content is stored in .versions directory
+		// For versioned buckets, all content is stored in .versions directory
 		// The latest version information is tracked in the .versions directory metadata
-
 		output = &CompleteMultipartUploadResult{
 			Location:  aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
 			Bucket:    input.Bucket,
-			ETag:      aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
+			ETag:      aws.String(etagQuote),
 			Key:       objectKey(input.Key),
 			VersionId: aws.String(versionId),
 		}
@@ -482,6 +488,8 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 				firstPartEntry := partEntries[completedPartNumbers[0]][0]
 				copySSEHeadersFromFirstPart(entry, firstPartEntry, "suspended versioning")
 			}
+			// Persist ETag to ensure subsequent HEAD/GET uses the same value
+			entry.Extended[s3_constants.ExtETagKey] = []byte(multipartETag)
 			if pentry.Attributes.Mime != "" {
 				entry.Attributes.Mime = pentry.Attributes.Mime
 			} else if mime != "" {
@@ -499,7 +507,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 		output = &CompleteMultipartUploadResult{
 			Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
 			Bucket:   input.Bucket,
-			ETag:     aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
+			ETag:     aws.String(etagQuote),
 			Key:      objectKey(input.Key),
 			// VersionId field intentionally omitted for suspended versioning
 		}
@@ -535,6 +543,8 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 				firstPartEntry := partEntries[completedPartNumbers[0]][0]
 				copySSEHeadersFromFirstPart(entry, firstPartEntry, "non-versioned")
 			}
+			// Persist ETag to ensure subsequent HEAD/GET uses the same value
+			entry.Extended[s3_constants.ExtETagKey] = []byte(multipartETag)
 			if pentry.Attributes.Mime != "" {
 				entry.Attributes.Mime = pentry.Attributes.Mime
 			} else if mime != "" {
@@ -556,7 +566,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
 		output = &CompleteMultipartUploadResult{
 			Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
 			Bucket:   input.Bucket,
-			ETag:     aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
+			ETag:     aws.String(etagQuote),
 			Key:      objectKey(input.Key),
 		}
 	}
@@ -929,3 +939,65 @@ func (s3a *S3ApiServer) applyMultipartEncryptionConfig(entry *filer_pb.Entry, co
 		glog.V(3).Infof("applyMultipartEncryptionConfig: applied SSE-S3 settings")
 	}
 }
+
+func sortEntriesByLatestChunk(entries []*filer_pb.Entry) {
+	slices.SortFunc(entries, func(a, b *filer_pb.Entry) int {
+		var aTs, bTs int64
+		if len(a.Chunks) > 0 {
+			aTs = a.Chunks[0].ModifiedTsNs
+		}
+		if len(b.Chunks) > 0 {
+			bTs = b.Chunks[0].ModifiedTsNs
+		}
+		return cmp.Compare(bTs, aTs)
+	})
+}
+
+func calculateMultipartETag(partEntries map[int][]*filer_pb.Entry, completedPartNumbers []int) string {
+	var etags []byte
+	for _, partNumber := range completedPartNumbers {
+		entries, ok := partEntries[partNumber]
+		if !ok || len(entries) == 0 {
+			continue
+		}
+		if len(entries) > 1 {
+			sortEntriesByLatestChunk(entries)
+		}
+		entry := entries[0]
+		etag := getEtagFromEntry(entry)
+		glog.V(4).Infof("calculateMultipartETag: part %d, entry %s, getEtagFromEntry result: %s", partNumber, entry.Name, etag)
+		etag = strings.Trim(etag, "\"")
+		if before, _, found := strings.Cut(etag, "-"); found {
+			etag = before
+		}
+		if etagBytes, err := hex.DecodeString(etag); err == nil {
+			etags = append(etags, etagBytes...)
+		} else {
+			glog.Warningf("calculateMultipartETag: failed to decode etag '%s' for part %d: %v", etag, partNumber, err)
+		}
+	}
+	return fmt.Sprintf("%x-%d", md5.Sum(etags), len(completedPartNumbers))
+}
+
+func getEtagFromEntry(entry *filer_pb.Entry) string {
+	if entry.Extended != nil {
+		if etagBytes, ok := entry.Extended[s3_constants.ExtETagKey]; ok {
+			etag := string(etagBytes)
+			if len(etag) > 0 {
+				if !strings.HasPrefix(etag, "\"") {
+					return "\"" + etag + "\""
+				}
+				return etag
+			}
+			// Empty stored ETag — fall through to filer.ETag calculation
+		}
+	}
+	// Fallback to filer.ETag which handles Attributes.Md5 consistently
+	etag := filer.ETag(entry)
+	entryName := entry.Name
+	if entryName == "" {
+		entryName = "entry"
+	}
+	glog.V(4).Infof("getEtagFromEntry: fallback to filer.ETag for %s: %s, chunkCount: %d", entryName, etag, len(entry.Chunks))
+	return "\"" + etag + "\""
+}

weed/s3api/iceberg/iceberg.go

@@ -18,37 +18,11 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
 )
 
-func (s *Server) checkAuth(w http.ResponseWriter, r *http.Request, action s3api.Action, bucketName string) bool {
-	identityName := s3_constants.GetIdentityNameFromContext(r)
-	if identityName == "" {
-		writeError(w, http.StatusUnauthorized, "NotAuthorizedException", "Authentication required")
-		return false
-	}
-
-	identityObj := s3_constants.GetIdentityFromContext(r)
-	if identityObj == nil {
-		writeError(w, http.StatusForbidden, "ForbiddenException", "Access denied: missing identity")
-		return false
-	}
-	identity, ok := identityObj.(*s3api.Identity)
-	if !ok {
-		writeError(w, http.StatusForbidden, "ForbiddenException", "Access denied: invalid identity")
-		return false
-	}
-
-	if !identity.CanDo(action, bucketName, "") {
-		writeError(w, http.StatusForbidden, "ForbiddenException", "Access denied")
-		return false
-	}
-	return true
-}
-
 // FilerClient provides access to the filer for storage operations.
 type FilerClient interface {
 	WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
@@ -79,17 +53,20 @@ func NewServer(filerClient FilerClient, authenticator S3Authenticator) *Server {
 
 // RegisterRoutes registers Iceberg REST API routes on the provided router.
 func (s *Server) RegisterRoutes(router *mux.Router) {
-	// Configuration endpoint
-	router.HandleFunc("/v1/config", s.Auth(s.handleConfig)).Methods(http.MethodGet)
+	// Add middleware to log all requests/responses
+	router.Use(loggingMiddleware)
 
-	// Namespace endpoints
+	// Configuration endpoint - no auth needed for config
+	router.HandleFunc("/v1/config", s.handleConfig).Methods(http.MethodGet)
+
+	// Namespace endpoints - wrapped with Auth middleware
 	router.HandleFunc("/v1/namespaces", s.Auth(s.handleListNamespaces)).Methods(http.MethodGet)
 	router.HandleFunc("/v1/namespaces", s.Auth(s.handleCreateNamespace)).Methods(http.MethodPost)
 	router.HandleFunc("/v1/namespaces/{namespace}", s.Auth(s.handleGetNamespace)).Methods(http.MethodGet)
 	router.HandleFunc("/v1/namespaces/{namespace}", s.Auth(s.handleNamespaceExists)).Methods(http.MethodHead)
 	router.HandleFunc("/v1/namespaces/{namespace}", s.Auth(s.handleDropNamespace)).Methods(http.MethodDelete)
 
-	// Table endpoints
+	// Table endpoints - wrapped with Auth middleware
 	router.HandleFunc("/v1/namespaces/{namespace}/tables", s.Auth(s.handleListTables)).Methods(http.MethodGet)
 	router.HandleFunc("/v1/namespaces/{namespace}/tables", s.Auth(s.handleCreateTable)).Methods(http.MethodPost)
 	router.HandleFunc("/v1/namespaces/{namespace}/tables/{table}", s.Auth(s.handleLoadTable)).Methods(http.MethodGet)
@@ -97,7 +74,7 @@ func (s *Server) RegisterRoutes(router *mux.Router) {
 	router.HandleFunc("/v1/namespaces/{namespace}/tables/{table}", s.Auth(s.handleDropTable)).Methods(http.MethodDelete)
 	router.HandleFunc("/v1/namespaces/{namespace}/tables/{table}", s.Auth(s.handleUpdateTable)).Methods(http.MethodPost)
 
-	// With prefix support
+	// With prefix support - wrapped with Auth middleware
 	router.HandleFunc("/v1/{prefix}/namespaces", s.Auth(s.handleListNamespaces)).Methods(http.MethodGet)
 	router.HandleFunc("/v1/{prefix}/namespaces", s.Auth(s.handleCreateNamespace)).Methods(http.MethodPost)
 	router.HandleFunc("/v1/{prefix}/namespaces/{namespace}", s.Auth(s.handleGetNamespace)).Methods(http.MethodGet)
@@ -110,7 +87,48 @@ func (s *Server) RegisterRoutes(router *mux.Router) {
 	router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables/{table}", s.Auth(s.handleDropTable)).Methods(http.MethodDelete)
 	router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables/{table}", s.Auth(s.handleUpdateTable)).Methods(http.MethodPost)
 
-	glog.V(0).Infof("Registered Iceberg REST Catalog routes")
+	// Catch-all for debugging
+	router.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		glog.V(2).Infof("Catch-all route hit: %s %s", r.Method, r.RequestURI)
+		writeError(w, http.StatusNotFound, "NotFound", "Path not found")
+	})
+
+	glog.V(2).Infof("Registered Iceberg REST Catalog routes")
+}
+
+func loggingMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		glog.V(2).Infof("Iceberg REST request: %s %s from %s", r.Method, r.RequestURI, r.RemoteAddr)
+
+		// Log all headers for debugging
+		glog.V(2).Infof("Iceberg REST headers:")
+		for name, values := range r.Header {
+			for _, value := range values {
+				// Redact sensitive headers
+				if name == "Authorization" && len(value) > 20 {
+					glog.V(2).Infof("  %s: %s...%s", name, value[:20], value[len(value)-10:])
+				} else {
+					glog.V(2).Infof("  %s: %s", name, value)
+				}
+			}
+		}
+
+		// Create a response writer that captures the status code
+		wrapped := &responseWriter{ResponseWriter: w}
+		next.ServeHTTP(wrapped, r)
+
+		glog.V(2).Infof("Iceberg REST response: %s %s -> %d", r.Method, r.RequestURI, wrapped.statusCode)
+	})
+}
+
+type responseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func (w *responseWriter) WriteHeader(code int) {
+	w.statusCode = code
+	w.ResponseWriter.WriteHeader(code)
 }
 
 func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
@@ -293,8 +311,8 @@ func getBucketFromPrefix(r *http.Request) string {
 	if prefix := vars["prefix"]; prefix != "" {
 		return prefix
 	}
-	// Default bucket if no prefix
-	return "default"
+	// Default bucket if no prefix - use "warehouse" for Iceberg
+	return "warehouse"
 }
 
 // buildTableBucketARN builds an ARN for a table bucket.
@@ -305,25 +323,28 @@ func buildTableBucketARN(bucketName string) string {
 
 // handleConfig returns catalog configuration.
 func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) {
-	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_READ, bucketName) {
-		return
-	}
+	glog.Infof("handleConfig: START")
+	glog.Infof("handleConfig: setting Content-Type header")
+	w.Header().Set("Content-Type", "application/json")
 	config := CatalogConfig{
 		Defaults:  map[string]string{},
 		Overrides: map[string]string{},
 	}
-	writeJSON(w, http.StatusOK, config)
+	glog.Infof("handleConfig: encoding JSON")
+	if err := json.NewEncoder(w).Encode(config); err != nil {
+		glog.Warningf("handleConfig: Failed to encode config: %v", err)
+	}
+	glog.Infof("handleConfig: COMPLETE")
 }
 
 // handleListNamespaces lists namespaces in a catalog.
 func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_LIST, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	// Use S3 Tables manager to list namespaces
 	var resp s3tables.ListNamespacesResponse
 	req := &s3tables.ListNamespacesRequest{
@@ -333,11 +354,11 @@ func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "ListNamespaces", req, &resp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "ListNamespaces", req, &resp, identityName)
 	})
 
 	if err != nil {
-		glog.V(1).Infof("Iceberg: ListNamespaces error: %v", err)
+		glog.Infof("Iceberg: ListNamespaces error: %v", err)
 		writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
 		return
 	}
@@ -357,11 +378,11 @@ func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
 // handleCreateNamespace creates a new namespace.
 func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_WRITE, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	var req CreateNamespaceRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 		writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid request body")
@@ -382,15 +403,18 @@ func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "CreateNamespace", createReq, &createResp, "")
+		glog.Errorf("Iceberg: handleCreateNamespace calling Execute with identityName=%s", identityName)
+		return s.tablesManager.Execute(r.Context(), mgrClient, "CreateNamespace", createReq, &createResp, identityName)
 	})
 
 	if err != nil {
+		glog.Errorf("Iceberg: handleCreateNamespace error: %v", err)
+
 		if strings.Contains(err.Error(), "already exists") {
 			writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
 			return
 		}
-		glog.V(1).Infof("Iceberg: CreateNamespace error: %v", err)
+		glog.Infof("Iceberg: CreateNamespace error: %v", err)
 		writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
 		return
 	}
@@ -418,11 +442,11 @@ func (s *Server) handleGetNamespace(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_READ, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	// Use S3 Tables manager to get namespace
 	getReq := &s3tables.GetNamespaceRequest{
 		TableBucketARN: bucketARN,
@@ -432,7 +456,7 @@ func (s *Server) handleGetNamespace(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, identityName)
 	})
 
 	if err != nil {
@@ -462,11 +486,11 @@ func (s *Server) handleNamespaceExists(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_READ, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	getReq := &s3tables.GetNamespaceRequest{
 		TableBucketARN: bucketARN,
 		Namespace:      namespace,
@@ -475,7 +499,7 @@ func (s *Server) handleNamespaceExists(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, identityName)
 	})
 
 	if err != nil {
@@ -500,11 +524,11 @@ func (s *Server) handleDropNamespace(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_DELETE_BUCKET, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	deleteReq := &s3tables.DeleteNamespaceRequest{
 		TableBucketARN: bucketARN,
 		Namespace:      namespace,
@@ -512,10 +536,11 @@ func (s *Server) handleDropNamespace(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "DeleteNamespace", deleteReq, nil, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "DeleteNamespace", deleteReq, nil, identityName)
 	})
 
 	if err != nil {
+
 		if strings.Contains(err.Error(), "not found") {
 			writeError(w, http.StatusNotFound, "NoSuchNamespaceException", fmt.Sprintf("Namespace does not exist: %v", namespace))
 			return
@@ -542,11 +567,11 @@ func (s *Server) handleListTables(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_LIST, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	listReq := &s3tables.ListTablesRequest{
 		TableBucketARN: bucketARN,
 		Namespace:      namespace,
@@ -556,7 +581,7 @@ func (s *Server) handleListTables(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "ListTables", listReq, &listResp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "ListTables", listReq, &listResp, identityName)
 	})
 
 	if err != nil {
@@ -605,11 +630,11 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_WRITE, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	// Generate UUID for the new table
 	tableUUID := uuid.New()
 	location := fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), req.Name)
@@ -657,7 +682,7 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 
 	err = s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "CreateTable", createReq, &createResp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "CreateTable", createReq, &createResp, identityName)
 	})
 
 	if err != nil {
@@ -696,11 +721,11 @@ func (s *Server) handleLoadTable(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_READ, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	getReq := &s3tables.GetTableRequest{
 		TableBucketARN: bucketARN,
 		Namespace:      namespace,
@@ -710,10 +735,11 @@ func (s *Server) handleLoadTable(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
 	})
 
 	if err != nil {
+
 		if strings.Contains(err.Error(), "not found") {
 			writeError(w, http.StatusNotFound, "NoSuchTableException", fmt.Sprintf("Table does not exist: %s", tableName))
 			return
@@ -771,11 +797,11 @@ func (s *Server) handleTableExists(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_READ, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	getReq := &s3tables.GetTableRequest{
 		TableBucketARN: bucketARN,
 		Namespace:      namespace,
@@ -785,7 +811,7 @@ func (s *Server) handleTableExists(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
 	})
 
 	if err != nil {
@@ -807,11 +833,11 @@ func (s *Server) handleDropTable(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_DELETE_BUCKET, bucketName) {
-		return
-	}
 	bucketARN := buildTableBucketARN(bucketName)
 
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+
 	deleteReq := &s3tables.DeleteTableRequest{
 		TableBucketARN: bucketARN,
 		Namespace:      namespace,
@@ -820,7 +846,7 @@ func (s *Server) handleDropTable(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "DeleteTable", deleteReq, nil, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "DeleteTable", deleteReq, nil, identityName)
 	})
 
 	if err != nil {
@@ -849,9 +875,10 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 	}
 
 	bucketName := getBucketFromPrefix(r)
-	if !s.checkAuth(w, r, s3_constants.ACTION_WRITE, bucketName) {
-		return
-	}
+	bucketARN := buildTableBucketARN(bucketName)
+
+	// Extract identity from context
+	identityName := s3_constants.GetIdentityNameFromContext(r)
 
 	// Parse the commit request
 	var req CommitTableRequest
@@ -860,8 +887,6 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bucketARN := buildTableBucketARN(bucketName)
-
 	// First, load current table metadata
 	getReq := &s3tables.GetTableRequest{
 		TableBucketARN: bucketARN,
@@ -872,7 +897,7 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 
 	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
 	})
 
 	if err != nil {
@@ -985,7 +1010,7 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 		// 1. Write metadata file (this would normally be an S3 PutObject,
 		// but s3tables manager handles the metadata storage logic)
 		// For now, we assume s3tables.UpdateTable handles the reference update.
-		return s.tablesManager.Execute(r.Context(), mgrClient, "UpdateTable", updateReq, nil, "")
+		return s.tablesManager.Execute(r.Context(), mgrClient, "UpdateTable", updateReq, nil, identityName)
 	})
 
 	if err != nil {
@@ -1002,14 +1027,6 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, result)
 }
 
-// loadTableResultJSON is used for JSON serialization of LoadTableResult.
-// It wraps table.Metadata (which is an interface) for proper JSON output.
-type loadTableResultJSON struct {
-	MetadataLocation string             `json:"metadata-location,omitempty"`
-	Metadata         table.Metadata     `json:"metadata"`
-	Config           iceberg.Properties `json:"config,omitempty"`
-}
-
 // newTableMetadata creates a new table.Metadata object with the given parameters.
 // Uses iceberg-go's MetadataBuilder pattern for proper spec compliance.
 func newTableMetadata(

weed/s3api/policy_engine/engine_test.go

@@ -286,6 +286,18 @@ func TestPolicyValidation(t *testing.T) {
 			}`,
 			expectError: false,
 		},
+		{
+			name: "Valid policy with single statement object",
+			policyJSON: `{
+				"Version": "2012-10-17",
+				"Statement": {
+					"Effect": "Allow",
+					"Action": "s3:GetObject",
+					"Resource": "arn:aws:s3:::test-bucket/*"
+				}
+			}`,
+			expectError: false,
+		},
 		{
 			name: "Invalid version",
 			policyJSON: `{

weed/s3api/policy_engine/types.go

@@ -88,6 +88,37 @@ type PolicyDocument struct {
 	Statement []PolicyStatement `json:"Statement"`
 }
 
+// UnmarshalJSON implements json.Unmarshaler for PolicyDocument
+func (p *PolicyDocument) UnmarshalJSON(data []byte) error {
+	type Alias PolicyDocument
+	aux := &struct {
+		Statement json.RawMessage `json:"Statement"`
+		*Alias
+	}{
+		Alias: (*Alias)(p),
+	}
+
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+
+	// Try unmarshaling as []PolicyStatement first
+	var statements []PolicyStatement
+	if err := json.Unmarshal(aux.Statement, &statements); err == nil {
+		p.Statement = statements
+		return nil
+	}
+
+	// Try unmarshaling as single PolicyStatement
+	var statement PolicyStatement
+	if err := json.Unmarshal(aux.Statement, &statement); err == nil {
+		p.Statement = []PolicyStatement{statement}
+		return nil
+	}
+
+	return fmt.Errorf("Statement must be an array or a single object")
+}
+
 // PolicyStatement represents a single policy statement
 type PolicyStatement struct {
 	Sid         string               `json:"Sid,omitempty"`

weed/s3api/s3_sse_c.go

@@ -263,18 +263,13 @@ func CreateSSECDecryptedReaderWithOffset(r io.Reader, customerKey *SSECustomerKe
 
 // createCTRStreamWithOffset creates a CTR stream positioned at a specific counter offset
 func createCTRStreamWithOffset(block cipher.Block, iv []byte, counterOffset uint64) cipher.Stream {
-	// Create a copy of the IV to avoid modifying the original
-	offsetIV := make([]byte, len(iv))
-	copy(offsetIV, iv)
-
-	// Calculate the counter offset in blocks (AES block size is 16 bytes)
-	blockOffset := counterOffset / 16
-
-	// Add the block offset to the counter portion of the IV
-	// In AES-CTR, the last 8 bytes of the IV are typically used as the counter
-	addCounterToIV(offsetIV, blockOffset)
-
-	return cipher.NewCTR(block, offsetIV)
+	adjustedIV, skip := calculateIVWithOffset(iv, int64(counterOffset))
+	stream := cipher.NewCTR(block, adjustedIV)
+	if skip > 0 {
+		dummy := make([]byte, skip)
+		stream.XORKeyStream(dummy, dummy)
+	}
+	return stream
 }
 
 // addCounterToIV adds a counter value to the IV (treating last 8 bytes as big-endian counter)

weed/s3api/s3_sse_multipart_test.go

@@ -422,6 +422,58 @@ func TestMultipartSSEMixedScenarios(t *testing.T) {
 	})
 }
 
+func TestSSECLargeObjectChunkReassembly(t *testing.T) {
+	keyPair := GenerateTestSSECKey(1)
+	customerKey := &SSECustomerKey{
+		Algorithm: "AES256",
+		Key:       keyPair.Key,
+		KeyMD5:    keyPair.KeyMD5,
+	}
+
+	const chunkSize = 8 * 1024 * 1024 // matches putToFiler chunk size
+	totalSize := chunkSize*2 + 3*1024*1024
+	plaintext := make([]byte, totalSize)
+	for i := range plaintext {
+		plaintext[i] = byte(i % 251)
+	}
+
+	encryptedReader, iv, err := CreateSSECEncryptedReader(bytes.NewReader(plaintext), customerKey)
+	if err != nil {
+		t.Fatalf("Failed to create encrypted reader: %v", err)
+	}
+	encryptedData, err := io.ReadAll(encryptedReader)
+	if err != nil {
+		t.Fatalf("Failed to read encrypted data: %v", err)
+	}
+
+	var reconstructed bytes.Buffer
+	offset := int64(0)
+	for offset < int64(len(encryptedData)) {
+		end := offset + chunkSize
+		if end > int64(len(encryptedData)) {
+			end = int64(len(encryptedData))
+		}
+
+		chunkIV := make([]byte, len(iv))
+		copy(chunkIV, iv)
+		chunkReader := bytes.NewReader(encryptedData[offset:end])
+		decryptedReader, decErr := CreateSSECDecryptedReaderWithOffset(chunkReader, customerKey, chunkIV, uint64(offset))
+		if decErr != nil {
+			t.Fatalf("Failed to create decrypted reader for offset %d: %v", offset, decErr)
+		}
+		decryptedChunk, decErr := io.ReadAll(decryptedReader)
+		if decErr != nil {
+			t.Fatalf("Failed to read decrypted chunk at offset %d: %v", offset, decErr)
+		}
+		reconstructed.Write(decryptedChunk)
+		offset = end
+	}
+
+	if !bytes.Equal(reconstructed.Bytes(), plaintext) {
+		t.Fatalf("Reconstructed data mismatch: expected %d bytes, got %d", len(plaintext), reconstructed.Len())
+	}
+}
+
 // TestMultipartSSEPerformance tests performance characteristics of SSE with multipart
 func TestMultipartSSEPerformance(t *testing.T) {
 	if testing.Short() {

weed/s3api/s3api_bucket_handlers.go

@@ -117,6 +117,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 		Buckets: listBuckets,
 	}
 
+	glog.V(3).Infof("ListBucketsHandler response: %+v", response)
 	writeSuccessResponseXML(w, r, response)
 }
 

weed/s3api/s3api_etag_quoting_test.go

@@ -0,0 +1,167 @@
+package s3api
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+// TestReproIfMatchMismatch tests specifically for the scenario where internal ETag
+// is unquoted (common in SeaweedFS) but client sends quoted ETag in If-Match.
+func TestReproIfMatchMismatch(t *testing.T) {
+	bucket := "test-bucket"
+	object := "/test-key"
+	etagValue := "37b51d194a7513e45b56f6524f2d51f2"
+
+	// Scenario 1: Internal ETag is UNQUOTED (stored in Extended), Client sends QUOTED If-Match
+	// This mirrors the behavior we enforced in filer_multipart.go
+	t.Run("UnquotedInternal_QuotedHeader", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Name: "test-key",
+			Extended: map[string][]byte{
+				s3_constants.ExtETagKey: []byte(etagValue), // Unquoted
+			},
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:    time.Now().Unix(),
+				FileSize: 1024,
+			},
+		}
+
+		getter := &MockEntryGetter{mockEntry: entry}
+		req := createTestGetRequest(bucket, object)
+		// Client sends quoted ETag
+		req.Header.Set(s3_constants.IfMatch, "\""+etagValue+"\"")
+
+		s3a := NewS3ApiServerForTest()
+		result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
+
+		if result.ErrorCode != s3err.ErrNone {
+			t.Errorf("Expected success (ErrNone) for unquoted internal ETag and quoted header, got %v. Internal ETag: %s", result.ErrorCode, string(entry.Extended[s3_constants.ExtETagKey]))
+		}
+	})
+
+	// Scenario 2: Internal ETag is QUOTED (stored in Extended), Client sends QUOTED If-Match
+	// This handles legacy or mixed content
+	t.Run("QuotedInternal_QuotedHeader", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Name: "test-key",
+			Extended: map[string][]byte{
+				s3_constants.ExtETagKey: []byte("\"" + etagValue + "\""), // Quoted
+			},
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:    time.Now().Unix(),
+				FileSize: 1024,
+			},
+		}
+
+		getter := &MockEntryGetter{mockEntry: entry}
+		req := createTestGetRequest(bucket, object)
+		req.Header.Set(s3_constants.IfMatch, "\""+etagValue+"\"")
+
+		s3a := NewS3ApiServerForTest()
+		result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
+
+		if result.ErrorCode != s3err.ErrNone {
+			t.Errorf("Expected success (ErrNone) for quoted internal ETag and quoted header, got %v", result.ErrorCode)
+		}
+	})
+
+	// Scenario 3: Internal ETag is from Md5 (QUOTED by getObjectETag), Client sends QUOTED If-Match
+	t.Run("Md5Internal_QuotedHeader", func(t *testing.T) {
+		// Mock Md5 attribute (16 bytes)
+		md5Bytes := make([]byte, 16)
+		copy(md5Bytes, []byte("1234567890123456")) // This doesn't match the hex string below, but getObjectETag formats it as hex
+
+		// Expected ETag from Md5 is hex string of bytes
+		expectedHex := fmt.Sprintf("%x", md5Bytes)
+
+		entry := &filer_pb.Entry{
+			Name: "test-key",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:    time.Now().Unix(),
+				FileSize: 1024,
+				Md5:      md5Bytes,
+			},
+		}
+
+		getter := &MockEntryGetter{mockEntry: entry}
+		req := createTestGetRequest(bucket, object)
+		req.Header.Set(s3_constants.IfMatch, "\""+expectedHex+"\"")
+
+		s3a := NewS3ApiServerForTest()
+		result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
+
+		if result.ErrorCode != s3err.ErrNone {
+			t.Errorf("Expected success (ErrNone) for Md5 internal ETag and quoted header, got %v", result.ErrorCode)
+		}
+	})
+
+	// Test getObjectETag specifically ensuring it returns quoted strings
+	t.Run("getObjectETag_ShouldReturnQuoted", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Name: "test-key",
+			Extended: map[string][]byte{
+				s3_constants.ExtETagKey: []byte("unquoted-etag"),
+			},
+		}
+
+		s3a := NewS3ApiServerForTest()
+		etag := s3a.getObjectETag(entry)
+
+		expected := "\"unquoted-etag\""
+		if etag != expected {
+			t.Errorf("Expected quoted ETag %s, got %s", expected, etag)
+		}
+	})
+
+	// Test getObjectETag fallback when Extended ETag is present but empty
+	t.Run("getObjectETag_EmptyExtended_ShouldFallback", func(t *testing.T) {
+		md5Bytes := []byte("1234567890123456")
+		expectedHex := fmt.Sprintf("\"%x\"", md5Bytes)
+
+		entry := &filer_pb.Entry{
+			Name: "test-key-fallback",
+			Extended: map[string][]byte{
+				s3_constants.ExtETagKey: []byte(""), // Present but empty
+			},
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:    time.Now().Unix(),
+				FileSize: 1024,
+				Md5:      md5Bytes,
+			},
+		}
+
+		s3a := NewS3ApiServerForTest()
+		etag := s3a.getObjectETag(entry)
+
+		if etag != expectedHex {
+			t.Errorf("Expected fallback ETag %s, got %s", expectedHex, etag)
+		}
+	})
+
+	// Test newListEntry ETag behavior
+	t.Run("newListEntry_ShouldReturnQuoted", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Name: "test-key",
+			Extended: map[string][]byte{
+				s3_constants.ExtETagKey: []byte("unquoted-etag"),
+			},
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:    time.Now().Unix(),
+				FileSize: 1024,
+			},
+		}
+
+		s3a := NewS3ApiServerForTest()
+		listEntry := newListEntry(s3a, entry, "", "bucket/dir", "test-key", "bucket/", false, false, false)
+
+		expected := "\"unquoted-etag\""
+		if listEntry.ETag != expected {
+			t.Errorf("Expected quoted ETag %s, got %s", expected, listEntry.ETag)
+		}
+	})
+}

weed/s3api/s3api_object_handlers.go

@@ -483,7 +483,7 @@ func (s3a *S3ApiServer) handleDirectoryObjectRequest(w http.ResponseWriter, r *h
 	return false // Not a directory object, continue with normal processing
 }
 
-func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bucketPrefix string, fetchOwner bool, isDirectory bool, encodingTypeUrl bool, iam AccountManager) (listEntry ListEntry) {
+func newListEntry(s3a *S3ApiServer, entry *filer_pb.Entry, key string, dir string, name string, bucketPrefix string, fetchOwner bool, isDirectory bool, encodingTypeUrl bool) (listEntry ListEntry) {
 	storageClass := "STANDARD"
 	if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
 		storageClass = string(v)
@@ -500,15 +500,7 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu
 	}
 	// Determine ETag: prioritize ExtETagKey for versioned objects (supports multipart ETags),
 	// then fall back to filer.ETag() which uses Md5 attribute or calculates from chunks
-	var etag string
-	if entry.Extended != nil {
-		if etagBytes, hasETag := entry.Extended[s3_constants.ExtETagKey]; hasETag {
-			etag = string(etagBytes)
-		}
-	}
-	if etag == "" {
-		etag = "\"" + filer.ETag(entry) + "\""
-	}
+	etag := s3a.getObjectETag(entry)
 	listEntry = ListEntry{
 		Key:          key,
 		LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
@@ -531,7 +523,7 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu
 			displayName = "anonymous"
 		} else {
 			// Get the proper display name from IAM system
-			displayName = iam.GetAccountNameById(ownerID)
+			displayName = s3a.iam.GetAccountNameById(ownerID)
 			// Fallback to ownerID if no display name found
 			if displayName == "" {
 				displayName = ownerID
@@ -1512,17 +1504,18 @@ func writeZeroBytes(w io.Writer, n int64) error {
 //
 // IV Handling for SSE-C:
 // ----------------------
-// SSE-C multipart encryption (see lines 2772-2781) differs fundamentally from SSE-KMS/SSE-S3:
+// SSE-C multipart encryption differs from SSE-KMS/SSE-S3:
 //
-// 1. Encryption: CreateSSECEncryptedReader generates a RANDOM IV per part/chunk
-//   - Each part starts with a fresh random IV
+// 1. Encryption: CreateSSECEncryptedReader generates a RANDOM IV per part
+//   - Each part starts with a fresh random IV (NOT derived from a base IV)
 //   - CTR counter starts from 0 for each part: counter₀, counter₁, counter₂, ...
-//   - PartOffset is stored in metadata but NOT applied during encryption
+//   - PartOffset is stored in metadata to describe where this chunk sits in that encrypted stream
 //
-// 2. Decryption: Use the stored IV directly WITHOUT offset adjustment
-//   - The stored IV already represents the start of this part's encryption
-//   - Applying calculateIVWithOffset would shift to counterₙ, misaligning the keystream
-//   - Result: XOR with wrong keystream = corrupted plaintext
+// 2. Decryption: Use the stored per-part IV and advance the CTR by PartOffset
+//   - CreateSSECDecryptedReaderWithOffset internally uses calculateIVWithOffset to advance
+//     the CTR counter to reach PartOffset within the per-part encrypted stream
+//   - calculateIVWithOffset is applied to the per-part IV, NOT to derive a global base IV
+//   - Do NOT compute a single base IV for all parts (unlike SSE-KMS/SSE-S3)
 //
 // This contrasts with SSE-KMS/SSE-S3 which use: base IV + calculateIVWithOffset(ChunkOffset)
 func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, customerKey *SSECustomerKey) (io.Reader, error) {
@@ -1544,11 +1537,14 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil
 			return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
 		}
 
-		// CRITICAL: Use stored IV directly WITHOUT offset adjustment
-		// The stored IV is the random IV used at encryption time for this specific part
-		// SSE-C does NOT apply calculateIVWithOffset during encryption, so we must not apply it during decryption
-		// (See documentation above and at lines 2772-2781 for detailed explanation)
-		decryptedReader, decryptErr := CreateSSECDecryptedReader(fullChunkReader, customerKey, chunkIV)
+		partOffset := ssecMetadata.PartOffset
+		if partOffset < 0 {
+			fullChunkReader.Close()
+			return nil, fmt.Errorf("invalid SSE-C part offset %d for chunk %s", partOffset, chunkView.FileId)
+		}
+
+		// Use stored IV and advance CTR stream by PartOffset within the encrypted stream
+		decryptedReader, decryptErr := CreateSSECDecryptedReaderWithOffset(fullChunkReader, customerKey, chunkIV, uint64(partOffset))
 		if decryptErr != nil {
 			fullChunkReader.Close()
 			return nil, fmt.Errorf("failed to create decrypted reader: %w", decryptErr)
@@ -1964,9 +1960,9 @@ func (s3a *S3ApiServer) setResponseHeaders(w http.ResponseWriter, r *http.Reques
 
 	// Set ETag (but don't overwrite if already set, e.g., for part-specific GET requests)
 	if w.Header().Get("ETag") == "" {
-		etag := filer.ETag(entry)
+		etag := s3a.getObjectETag(entry)
 		if etag != "" {
-			w.Header().Set("ETag", "\""+etag+"\"")
+			w.Header().Set("ETag", etag)
 		}
 	}
 
@@ -2844,15 +2840,20 @@ func (s3a *S3ApiServer) createMultipartSSECDecryptedReaderDirect(ctx context.Con
 
 			// Note: SSE-C multipart behavior (differs from SSE-KMS/SSE-S3):
 			// - Upload: CreateSSECEncryptedReader generates RANDOM IV per part (no base IV + offset)
-			// - Metadata: PartOffset is stored but not used during encryption
-			// - Decryption: Use stored random IV directly (no offset adjustment needed)
+			// - Metadata: PartOffset tracks position within the encrypted stream
+			// - Decryption: Use stored IV and advance CTR stream by PartOffset
 			//
 			// This differs from:
 			// - SSE-KMS/SSE-S3: Use base IV + calculateIVWithOffset(partOffset) during encryption
 			// - CopyObject: Applies calculateIVWithOffset to SSE-C (which may be incorrect)
 			//
 			// TODO: Investigate CopyObject SSE-C PartOffset handling for consistency
-			decryptedChunkReader, decErr := CreateSSECDecryptedReader(chunkReader, customerKey, chunkIV)
+			partOffset := ssecMetadata.PartOffset
+			if partOffset < 0 {
+				chunkReader.Close()
+				return nil, fmt.Errorf("invalid SSE-C part offset %d for chunk %s", partOffset, chunk.GetFileIdString())
+			}
+			decryptedChunkReader, decErr := CreateSSECDecryptedReaderWithOffset(chunkReader, customerKey, chunkIV, uint64(partOffset))
 			if decErr != nil {
 				chunkReader.Close()
 				return nil, fmt.Errorf("failed to decrypt chunk: %v", decErr)
@@ -3235,26 +3236,32 @@ func (s3a *S3ApiServer) createMultipartSSECDecryptedReader(r *http.Request, prox
 				// Deserialize the SSE-C metadata stored in the unified metadata field
 				ssecMetadata, decErr := DeserializeSSECMetadata(chunk.GetSseMetadata())
 				if decErr != nil {
+					chunkReader.Close()
 					return nil, fmt.Errorf("failed to deserialize SSE-C metadata for chunk %s: %v", chunk.GetFileIdString(), decErr)
 				}
 
 				// Decode the IV from the metadata
 				iv, ivErr := base64.StdEncoding.DecodeString(ssecMetadata.IV)
 				if ivErr != nil {
+					chunkReader.Close()
 					return nil, fmt.Errorf("failed to decode IV for SSE-C chunk %s: %v", chunk.GetFileIdString(), ivErr)
 				}
 
-				// Note: For multipart SSE-C, each part was encrypted with offset=0
-				// So we use the stored IV directly without offset adjustment
-				// PartOffset is stored for informational purposes, but encryption uses offset=0
-				chunkIV := iv
+				partOffset := ssecMetadata.PartOffset
+				if partOffset < 0 {
+					chunkReader.Close()
+					return nil, fmt.Errorf("invalid SSE-C part offset %d for chunk %s", partOffset, chunk.GetFileIdString())
+				}
 
-				decryptedReader, decErr := CreateSSECDecryptedReader(chunkReader, customerKey, chunkIV)
+				// Use stored IV and advance CTR stream by PartOffset within the encrypted stream
+				decryptedReader, decErr := CreateSSECDecryptedReaderWithOffset(chunkReader, customerKey, iv, uint64(partOffset))
 				if decErr != nil {
+					chunkReader.Close()
 					return nil, fmt.Errorf("failed to create SSE-C decrypted reader for chunk %s: %v", chunk.GetFileIdString(), decErr)
 				}
 				readers = append(readers, decryptedReader)
 			} else {
+				chunkReader.Close()
 				return nil, fmt.Errorf("SSE-C chunk %s missing required metadata", chunk.GetFileIdString())
 			}
 		} else {

weed/s3api/s3api_object_handlers_list.go

@@ -109,6 +109,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 		responseV2.EncodingType = s3.EncodingTypeUrl
 	}
 
+	glog.V(3).Infof("ListObjectsV2Handler response: %+v", responseV2)
 	writeSuccessResponseXML(w, r, responseV2)
 }
 
@@ -155,6 +156,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
+	glog.V(3).Infof("ListObjectsV1Handler response: %+v", response)
 	writeSuccessResponseXML(w, r, response)
 }
 
@@ -200,6 +202,27 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 		var lastEntryWasCommonPrefix bool
 		var lastCommonPrefixName string
 
+		// Hoist versioning check out of per-entry callback
+		versioningState, _ := s3a.getVersioningState(bucket)
+		versioningEnabled := versioningState == "Enabled"
+
+		// Helper function to handle dedup/append logic
+		appendOrDedup := func(newEntry ListEntry) {
+			if versioningEnabled {
+				// For versioned buckets, we need to handle duplicates between the main file and the .versions directory
+				if len(contents) > 0 && contents[len(contents)-1].Key == newEntry.Key {
+					glog.V(3).Infof("listFilerEntries deduplicating versioned entry: %s", newEntry.Key)
+					contents[len(contents)-1] = newEntry
+				} else {
+					contents = append(contents, newEntry)
+					cursor.maxKeys--
+				}
+			} else {
+				contents = append(contents, newEntry)
+				cursor.maxKeys--
+			}
+		}
+
 		for {
 			empty := true
 
@@ -219,19 +242,13 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 						undelimitedPath = strings.TrimPrefix(undelimitedPath, originalPrefix)
 
 						delimitedPath := strings.SplitN(undelimitedPath, delimiter, 2)
-
 						if len(delimitedPath) == 2 {
 							// S3 clients expect the delimited prefix to contain the delimiter and prefix.
 							delimitedPrefix := originalPrefix + delimitedPath[0] + delimiter
 
-							for i := range commonPrefixes {
-								if commonPrefixes[i].Prefix == delimitedPrefix {
-									delimiterFound = true
-									break
-								}
-							}
-
-							if !delimiterFound {
+							// Check if this CommonPrefix already exists
+							if !lastEntryWasCommonPrefix || lastCommonPrefixName != delimitedPath[0] {
+								// New CommonPrefix found
 								commonPrefixes = append(commonPrefixes, PrefixEntry{
 									Prefix: delimitedPrefix,
 								})
@@ -247,14 +264,14 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 
 						// If no delimiter found in the directory object name, treat it as a regular key
 						if !delimiterFound {
-							contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, true, false, s3a.iam))
-							cursor.maxKeys--
+							newEntry := newListEntry(s3a, entry, "", dirName, entryName, bucketPrefix, fetchOwner, true, false)
+							appendOrDedup(newEntry)
 							lastEntryWasCommonPrefix = false
 						}
 					} else if entry.IsDirectoryKeyObject() {
 						// No delimiter specified, or delimiter doesn't apply - treat as regular key
-						contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, true, false, s3a.iam))
-						cursor.maxKeys--
+						newEntry := newListEntry(s3a, entry, "", dirName, entryName, bucketPrefix, fetchOwner, true, false)
+						appendOrDedup(newEntry)
 						lastEntryWasCommonPrefix = false
 						// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
 					} else if delimiter != "" { // A response can contain CommonPrefixes only if you specify a delimiter.
@@ -308,8 +325,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 					}
 					if !delimiterFound {
 						glog.V(4).Infof("Adding file to contents: %s", entryName)
-						contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, false, false, s3a.iam))
-						cursor.maxKeys--
+						newEntry := newListEntry(s3a, entry, "", dirName, entryName, bucketPrefix, fetchOwner, false, false)
+						appendOrDedup(newEntry)
 						lastEntryWasCommonPrefix = false
 					}
 				}

weed/s3api/s3api_object_handlers_put.go

@@ -540,6 +540,9 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
 	etag = filer.ETag(entry)
 	glog.V(4).Infof("putToFiler: Calculated ETag=%s for %d chunks", etag, len(chunkResult.FileChunks))
 
+	// Store ETag in Extended attribute for future retrieval (e.g. multipart parts)
+	entry.Extended[s3_constants.ExtETagKey] = []byte(etag)
+
 	// Set object owner
 	amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
 	if amzAccountId != "" {
@@ -1072,10 +1075,7 @@ func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object strin
 	}
 	versionEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(versionId)
 
-	// Store ETag with quotes for S3 compatibility
-	if !strings.HasPrefix(etag, "\"") {
-		etag = "\"" + etag + "\""
-	}
+	// Store ETag (unquoted) in Extended attribute
 	versionEntry.Extended[s3_constants.ExtETagKey] = []byte(etag)
 
 	// Set object owner for versioned objects
@@ -1594,7 +1594,14 @@ func parseConditionalHeaders(r *http.Request) (conditionalHeaders, s3err.ErrorCo
 func (s3a *S3ApiServer) getObjectETag(entry *filer_pb.Entry) string {
 	// Try to get ETag from Extended attributes first
 	if etagBytes, hasETag := entry.Extended[s3_constants.ExtETagKey]; hasETag {
-		return string(etagBytes)
+		etag := string(etagBytes)
+		if len(etag) > 0 {
+			if !strings.HasPrefix(etag, "\"") {
+				return "\"" + etag + "\""
+			}
+			return etag
+		}
+		// Empty stored ETag — fall through to Md5/chunk-based calculation
 	}
 	// Check for Md5 in Attributes (matches filer.ETag behavior)
 	// Note: len(nil slice) == 0 in Go, so no need for explicit nil check
@@ -1635,7 +1642,6 @@ func (s3a *S3ApiServer) validateConditionalHeaders(r *http.Request, headers cond
 	// 1. Check If-Match
 	if headers.ifMatch != "" {
 		if !objectExists {
-			glog.V(3).Infof("validateConditionalHeaders: If-Match failed - object %s/%s does not exist", bucket, object)
 			return s3err.ErrPreconditionFailed
 		}
 		// If `ifMatch` is "*", the condition is met if the object exists.
@@ -1645,7 +1651,6 @@ func (s3a *S3ApiServer) validateConditionalHeaders(r *http.Request, headers cond
 			objectETag := s3a.getObjectETag(entry)
 			// Use production etagMatches method
 			if !s3a.etagMatches(headers.ifMatch, objectETag) {
-				glog.V(3).Infof("validateConditionalHeaders: If-Match failed for object %s/%s - expected ETag %s, got %s", bucket, object, headers.ifMatch, objectETag)
 				return s3err.ErrPreconditionFailed
 			}
 		}
@@ -1787,7 +1792,7 @@ func (s3a *S3ApiServer) validateConditionalHeadersForReads(r *http.Request, head
 			objectETag := s3a.getObjectETag(entry)
 			// Use production etagMatches method
 			if !s3a.etagMatches(headers.ifMatch, objectETag) {
-				glog.V(3).Infof("validateConditionalHeadersForReads: If-Match failed for object %s/%s - expected ETag %s, got %s", bucket, object, headers.ifMatch, objectETag)
+				glog.V(3).Infof("validateConditionalHeadersForReads: If-Match failed for object %s/%s - header If-Match: [%s], object ETag: [%s]", bucket, object, headers.ifMatch, objectETag)
 				return ConditionalHeaderResult{ErrorCode: s3err.ErrPreconditionFailed, Entry: entry}
 			}
 		}

weed/s3api/s3api_object_handlers_test.go

@@ -26,11 +26,13 @@ func (m *mockAccountManager) GetAccountIdByEmail(email string) string {
 }
 
 func TestNewListEntryOwnerDisplayName(t *testing.T) {
-	// Create mock IAM with test accounts
-	iam := &mockAccountManager{
-		accounts: map[string]string{
-			"testid":    "M. Tester",
-			"userid123": "John Doe",
+	// Create S3ApiServer with a properly initialized IAM
+	s3a := &S3ApiServer{
+		iam: &IdentityAccessManagement{
+			accounts: map[string]*Account{
+				"testid":    {Id: "testid", DisplayName: "M. Tester"},
+				"userid123": {Id: "userid123", DisplayName: "John Doe"},
+			},
 		},
 	}
 
@@ -47,7 +49,7 @@ func TestNewListEntryOwnerDisplayName(t *testing.T) {
 	}
 
 	// Test that display name is correctly looked up from IAM
-	listEntry := newListEntry(entry, "", "dir", "test-object", "/buckets/test/", true, false, false, iam)
+	listEntry := newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", true, false, false)
 
 	assert.NotNil(t, listEntry.Owner, "Owner should be set when fetchOwner is true")
 	assert.Equal(t, "testid", listEntry.Owner.ID, "Owner ID should match stored owner")
@@ -55,20 +57,20 @@ func TestNewListEntryOwnerDisplayName(t *testing.T) {
 
 	// Test with owner that doesn't exist in IAM (should fallback to ID)
 	entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte("unknown-user")
-	listEntry = newListEntry(entry, "", "dir", "test-object", "/buckets/test/", true, false, false, iam)
+	listEntry = newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", true, false, false)
 
 	assert.Equal(t, "unknown-user", listEntry.Owner.ID, "Owner ID should match stored owner")
 	assert.Equal(t, "unknown-user", listEntry.Owner.DisplayName, "Display name should fallback to ID when not found in IAM")
 
 	// Test with no owner metadata (should use anonymous)
 	entry.Extended = make(map[string][]byte)
-	listEntry = newListEntry(entry, "", "dir", "test-object", "/buckets/test/", true, false, false, iam)
+	listEntry = newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", true, false, false)
 
 	assert.Equal(t, s3_constants.AccountAnonymousId, listEntry.Owner.ID, "Should use anonymous ID when no owner metadata")
 	assert.Equal(t, "anonymous", listEntry.Owner.DisplayName, "Should use anonymous display name when no owner metadata")
 
 	// Test with fetchOwner false (should not set owner)
-	listEntry = newListEntry(entry, "", "dir", "test-object", "/buckets/test/", false, false, false, iam)
+	listEntry = newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", false, false, false)
 
 	assert.Nil(t, listEntry.Owner, "Owner should not be set when fetchOwner is false")
 }

weed/s3api/s3api_object_versioning.go

@@ -640,6 +640,24 @@ func (vc *versionCollector) collectVersions(currentPath, relativePath string) er
 				continue
 			}
 
+			// Handle special directories that should bypass delimiter logic
+			// This ensures .versions directories are processed as version containers
+			// rather than being rolled up into CommonPrefixes when a delimiter is used
+			if entry.IsDirectory {
+				// Skip .uploads directory
+				if strings.HasPrefix(entry.Name, s3_constants.MultipartUploadsFolder) {
+					continue
+				}
+
+				// Handle .versions directory
+				if strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) {
+					if err := vc.processVersionsDirectory(entryPath); err != nil {
+						return err
+					}
+					continue
+				}
+			}
+
 			// Group into common prefixes if delimiter is found after the prefix
 			if vc.delimiter != "" {
 				fullKey := entryPath
@@ -688,16 +706,6 @@ func (vc *versionCollector) collectVersions(currentPath, relativePath string) er
 
 // processDirectory handles directory entries
 func (vc *versionCollector) processDirectory(currentPath, entryPath string, entry *filer_pb.Entry) error {
-	// Skip .uploads directory
-	if strings.HasPrefix(entry.Name, ".uploads") {
-		return nil
-	}
-
-	// Handle .versions directory
-	if strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) {
-		return vc.processVersionsDirectory(entryPath)
-	}
-
 	// Handle explicit S3 directory object
 	if entry.Attributes.Mime == s3_constants.FolderMimeType {
 		vc.processExplicitDirectory(entryPath, entry)
@@ -810,6 +818,9 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
 				// Try to get ETag from Extended attributes first
 				if etagBytes, hasETag := entry.Extended[s3_constants.ExtETagKey]; hasETag {
 					version.ETag = string(etagBytes)
+					if !strings.HasPrefix(version.ETag, "\"") {
+						version.ETag = "\"" + version.ETag + "\""
+					}
 				} else {
 					// Fallback: calculate ETag from chunks
 					version.ETag = s3a.calculateETagFromChunks(entry.Chunks)
@@ -1094,6 +1105,7 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
 	// Set the original prefix in the response (not the normalized internal prefix)
 	result.Prefix = originalPrefix
 
+	glog.V(3).Infof("ListObjectVersionsHandler response: %+v", result)
 	writeSuccessResponseXML(w, r, result)
 }
 

weed/s3api/s3api_tables.go

@@ -632,6 +632,7 @@ func buildUntagResourceRequest(r *http.Request) (interface{}, error) {
 // which performs granular permission checks based on the specific operation.
 func (s3a *S3ApiServer) authenticateS3Tables(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		glog.V(2).Infof("S3Tables: authenticateS3Tables called, iam.isEnabled()=%t", s3a.iam.isEnabled())
 		if !s3a.iam.isEnabled() {
 			f(w, r)
 			return
@@ -640,15 +641,19 @@ func (s3a *S3ApiServer) authenticateS3Tables(f http.HandlerFunc) http.HandlerFun
 		// Use AuthSignatureOnly to authenticate the request without authorizing specific actions
 		identity, errCode := s3a.iam.AuthSignatureOnly(r)
 		if errCode != s3err.ErrNone {
+			glog.Errorf("S3Tables: AuthSignatureOnly failed: %v", errCode)
 			s3err.WriteErrorResponse(w, r, errCode)
 			return
 		}
 
 		// Store the authenticated identity in request context
 		if identity != nil && identity.Name != "" {
+			glog.V(2).Infof("S3Tables: authenticated identity Name=%s Account.Id=%s", identity.Name, identity.Account.Id)
 			ctx := s3_constants.SetIdentityNameInContext(r.Context(), identity.Name)
 			ctx = s3_constants.SetIdentityInContext(ctx, identity)
 			r = r.WithContext(ctx)
+		} else {
+			glog.V(2).Infof("S3Tables: authenticated identity is nil or empty name")
 		}
 
 		f(w, r)

weed/s3api/s3tables/handler.go

@@ -164,9 +164,32 @@ func (h *S3TablesHandler) HandleRequest(w http.ResponseWriter, r *http.Request,
 // This is also used as the principal for permission checks, ensuring alignment between
 // the caller identity and ownership verification when IAM is enabled.
 func (h *S3TablesHandler) getAccountID(r *http.Request) string {
+	identityRaw := s3_constants.GetIdentityFromContext(r)
+	if identityRaw != nil {
+		// Use reflection to access the Account.Id field to avoid import cycle
+		val := reflect.ValueOf(identityRaw)
+		if val.Kind() == reflect.Ptr {
+			val = val.Elem()
+		}
+		if val.Kind() == reflect.Struct {
+			accountField := val.FieldByName("Account")
+			if accountField.IsValid() && !accountField.IsNil() {
+				accountVal := accountField.Elem()
+				if accountVal.Kind() == reflect.Struct {
+					idField := accountVal.FieldByName("Id")
+					if idField.IsValid() && idField.Kind() == reflect.String {
+						id := idField.String()
+						return id
+					}
+				}
+			}
+		}
+	}
+
 	if identityName := s3_constants.GetIdentityNameFromContext(r); identityName != "" {
 		return identityName
 	}
+
 	if accountID := r.Header.Get(s3_constants.AmzAccountId); accountID != "" {
 		return accountID
 	}

weed/s3api/s3tables/handler_namespace.go

@@ -9,13 +9,16 @@ import (
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 )
 
 // handleCreateNamespace creates a new namespace in a table bucket
 func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
+	glog.Errorf("S3Tables: handleCreateNamespace called")
 	var req CreateNamespaceRequest
 	if err := h.readRequestBody(r, &req); err != nil {
+		glog.Errorf("S3Tables: handleCreateNamespace failed to read request body: %v", err)
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
 		return err
 	}
@@ -83,12 +86,14 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
 	bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
 	principal := h.getAccountID(r)
 	identityActions := getIdentityActions(r)
+	glog.Infof("S3Tables: CreateNamespace permission check - principal=%s, owner=%s, actions=%v", principal, bucketMetadata.OwnerAccountID, identityActions)
 	if !CheckPermissionWithContext("CreateNamespace", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
 		Namespace:       namespaceName,
 		TableBucketTags: bucketTags,
 		IdentityActions: identityActions,
 	}) {
+		glog.Infof("S3Tables: Permission denied for CreateNamespace - principal=%s, owner=%s", principal, bucketMetadata.OwnerAccountID)
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
 		return ErrAccessDenied
 	}

weed/s3api/s3tables/permissions.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 )
@@ -110,6 +111,8 @@ func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, res
 		return true
 	}
 
+	glog.V(2).Infof("S3Tables: CheckPermission operation=%s principal=%s owner=%s", operation, principal, owner)
+
 	return checkPermission(operation, principal, owner, resourcePolicy, resourceARN, ctx)
 }
 

weed/server/filer_server_handlers_write_upload.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/md5"
+	"encoding/base64"
 	"fmt"
 	"hash"
 	"io"
@@ -128,16 +129,15 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request
 			}
 			if chunks != nil {
 				fileChunksLock.Lock()
-				fileChunksSize := len(fileChunks) + len(chunks)
 				for _, chunk := range chunks {
 					fileChunks = append(fileChunks, chunk)
-					glog.V(4).InfofCtx(ctx, "uploaded %s chunk %d to %s [%d,%d)", fileName, fileChunksSize, chunk.FileId, offset, offset+int64(chunk.Size))
 				}
 				fileChunksLock.Unlock()
 			}
 		}(chunkOffset, bytesBuffer)
 
 		// reset variables for the next chunk
+		glog.V(4).Infof("uploadReaderToChunks read chunk at offset %d, size %d", chunkOffset, dataSize)
 		chunkOffset = chunkOffset + dataSize
 
 		// if last chunk was not at full chunk size, but already exhausted the reader
@@ -162,7 +162,7 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request
 	return fileChunks, md5Hash, chunkOffset, nil, smallContent
 }
 
-func (fs *FilerServer) doUpload(ctx context.Context, urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
+func (fs *FilerServer) doUpload(ctx context.Context, urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt, contentMd5 string) (*operation.UploadResult, error, []byte) {
 
 	stats.FilerHandlerCounter.WithLabelValues(stats.ChunkUpload).Inc()
 	start := time.Now()
@@ -178,6 +178,7 @@ func (fs *FilerServer) doUpload(ctx context.Context, urlLocation string, limited
 		MimeType:          contentType,
 		PairMap:           pairMap,
 		Jwt:               auth,
+		Md5:               contentMd5,
 	}
 
 	uploader, err := operation.NewUploader()
@@ -217,8 +218,10 @@ func (fs *FilerServer) dataToChunkWithSSE(ctx context.Context, r *http.Request,
 			stats.FilerHandlerCounter.WithLabelValues(stats.ChunkAssignRetry).Inc()
 			return uploadErr
 		}
+		chunkMd5 := md5.Sum(data)
+		chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
 		// upload the chunk to the volume server
-		uploadResult, uploadErr, _ = fs.doUpload(ctx, urlLocation, dataReader, fileName, contentType, nil, auth)
+		uploadResult, uploadErr, _ = fs.doUpload(ctx, urlLocation, dataReader, fileName, contentType, nil, auth, chunkMd5B64)
 		if uploadErr != nil {
 			glog.V(4).InfofCtx(ctx, "retry later due to upload error: %v", uploadErr)
 			stats.FilerHandlerCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc()

weed/server/master_grpc_server_assign.go

@@ -79,6 +79,16 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 	}
 
 	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType)
+	if req.DiskType == "" {
+		if writable, _ := vl.GetWritableVolumeCount(); writable == 0 {
+			if hddVl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, types.ToDiskType(types.HddType)); hddVl != nil {
+				if writable, _ := hddVl.GetWritableVolumeCount(); writable > 0 {
+					option.DiskType = types.ToDiskType(types.HddType)
+					vl = hddVl
+				}
+			}
+		}
+	}
 	vl.SetLastGrowCount(req.WritableVolumeCount)
 
 	var (

weed/server/volume_grpc_admin.go

@@ -273,7 +273,7 @@ func (vs *VolumeServer) VolumeStatus(ctx context.Context, req *volume_server_pb.
 func (vs *VolumeServer) VolumeServerStatus(ctx context.Context, req *volume_server_pb.VolumeServerStatusRequest) (*volume_server_pb.VolumeServerStatusResponse, error) {
 
 	resp := &volume_server_pb.VolumeServerStatusResponse{
-		State:        vs.store.State.Pb,
+		State:        vs.store.State.Proto(),
 		MemoryStatus: stats.MemStat(),
 		Version:      version.Version(),
 		DataCenter:   vs.dataCenter,

weed/server/volume_grpc_scrub.go

@@ -31,11 +31,13 @@ func (vs *VolumeServer) ScrubVolume(ctx context.Context, req *volume_server_pb.S
 			return nil, fmt.Errorf("volume id %d not found", vid)
 		}
 
-		var files uint64
+		var files int64
 		var serrs []error
 		switch m := req.GetMode(); m {
 		case volume_server_pb.VolumeScrubMode_INDEX:
-			files, serrs = scrubVolumeIndex(ctx, v)
+			files, serrs = v.CheckIndex()
+		case volume_server_pb.VolumeScrubMode_LOCAL:
+			files, serrs = scrubVolumeLocal(ctx, v)
 		case volume_server_pb.VolumeScrubMode_FULL:
 			files, serrs = scrubVolumeFull(ctx, v)
 		default:
@@ -43,7 +45,7 @@ func (vs *VolumeServer) ScrubVolume(ctx context.Context, req *volume_server_pb.S
 		}
 
 		totalVolumes += 1
-		totalFiles += files
+		totalFiles += uint64(files)
 		if len(serrs) != 0 {
 			brokenVolumeIds = append(brokenVolumeIds, uint32(vid))
 			for _, err := range serrs {
@@ -61,12 +63,12 @@ func (vs *VolumeServer) ScrubVolume(ctx context.Context, req *volume_server_pb.S
 	return res, nil
 }
 
-func scrubVolumeIndex(ctx context.Context, v *storage.Volume) (uint64, []error) {
-	return 0, []error{fmt.Errorf("scrubVolumeIndex(): not implemented")}
+func scrubVolumeLocal(ctx context.Context, v *storage.Volume) (int64, []error) {
+	return 0, []error{fmt.Errorf("scrubVolumeLocal(): not implemented, see https://github.com/seaweedfs/seaweedfs/issues/8018")}
 }
 
-func scrubVolumeFull(ctx context.Context, v *storage.Volume) (uint64, []error) {
-	return 0, []error{fmt.Errorf("scrubVolumeFull(): not implemented")}
+func scrubVolumeFull(ctx context.Context, v *storage.Volume) (int64, []error) {
+	return 0, []error{fmt.Errorf("scrubVolumeFull(): not implemented, see https://github.com/seaweedfs/seaweedfs/issues/8018")}
 }
 
 func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb.ScrubEcVolumeRequest) (*volume_server_pb.ScrubEcVolumeResponse, error) {
@@ -91,12 +93,15 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
 			return nil, fmt.Errorf("EC volume id %d not found", vid)
 		}
 
-		var files uint64
+		var files int64
 		var shardInfos []*volume_server_pb.EcShardInfo
 		var serrs []error
 		switch m := req.GetMode(); m {
 		case volume_server_pb.VolumeScrubMode_INDEX:
-			files, shardInfos, serrs = scrubEcVolumeIndex(v)
+			// index scrubs do not verify individual EC shards
+			files, serrs = v.CheckIndex()
+		case volume_server_pb.VolumeScrubMode_LOCAL:
+			files, shardInfos, serrs = scrubEcVolumeLocal(ctx, v)
 		case volume_server_pb.VolumeScrubMode_FULL:
 			files, shardInfos, serrs = scrubEcVolumeFull(ctx, v)
 		default:
@@ -104,7 +109,7 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
 		}
 
 		totalVolumes += 1
-		totalFiles += files
+		totalFiles += uint64(files)
 		if len(serrs) != 0 || len(shardInfos) != 0 {
 			brokenVolumeIds = append(brokenVolumeIds, uint32(vid))
 			brokenShardInfos = append(brokenShardInfos, shardInfos...)
@@ -124,10 +129,10 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
 	return res, nil
 }
 
-func scrubEcVolumeIndex(ecv *erasure_coding.EcVolume) (uint64, []*volume_server_pb.EcShardInfo, []error) {
-	return 0, nil, []error{fmt.Errorf("scrubEcVolumeIndex(): not implemented")}
+func scrubEcVolumeLocal(ctx context.Context, v *erasure_coding.EcVolume) (int64, []*volume_server_pb.EcShardInfo, []error) {
+	return 0, nil, []error{fmt.Errorf("scrubEcVolumeLocal(): not implemented, see https://github.com/seaweedfs/seaweedfs/issues/8018")}
 }
 
-func scrubEcVolumeFull(ctx context.Context, v *erasure_coding.EcVolume) (uint64, []*volume_server_pb.EcShardInfo, []error) {
-	return 0, nil, []error{fmt.Errorf("scrubEcVolumeFull(): not implemented")}
+func scrubEcVolumeFull(ctx context.Context, v *erasure_coding.EcVolume) (int64, []*volume_server_pb.EcShardInfo, []error) {
+	return 0, nil, []error{fmt.Errorf("scrubEcVolumeFull(): not implemented, see https://github.com/seaweedfs/seaweedfs/issues/8018")}
 }