3.91

Support filtering source disk type in volume.tier.upload (#6868 )
Fix wrong error handling in volume.tier.upload when stream == nil but copyErr != nil (#6867 )
2026-10-02 04:32:42 +00:00 · 2025-06-15 20:42:16 -07:00 · 2025-06-15 20:30:04 -07:00 · 2025-06-15 20:28:40 -07:00 · 2025-06-15 20:27:48 -07:00 · 2025-06-13 12:27:39 -07:00
78 changed files with 2449 additions and 862 deletions
@@ -53,7 +53,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed-large-disk
@@ -68,7 +68,7 @@ jobs:
          release_tag: dev
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed-normal-disk
@@ -102,7 +102,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed-large-disk
@@ -117,7 +117,7 @@ jobs:
          release_tag: dev
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed-normal-disk
@@ -38,7 +38,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -52,7 +52,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -38,7 +38,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -52,7 +52,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -38,7 +38,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -52,7 +52,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -38,7 +38,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -52,7 +52,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -39,7 +39,7 @@ jobs:
          build_flags: -tags elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -53,7 +53,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -38,7 +38,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -52,7 +52,7 @@ jobs:
          overwrite: true
          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
          binary_name: weed
@@ -6,7 +6,7 @@ ARG BRANCH=${BRANCH:-master}
 ARG TAGS
 RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
  && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"

 FROM alpine AS final
@@ -3,7 +3,7 @@ FROM golang:1.24 as builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev

-ENV ROCKSDB_VERSION v9.10.0
+ENV ROCKSDB_VERSION v10.2.1

 # build RocksDB
 RUN cd /tmp && \
@@ -3,7 +3,7 @@ FROM golang:1.24 as builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev

-ENV ROCKSDB_VERSION v9.10.0
+ENV ROCKSDB_VERSION v10.2.1

 # build RocksDB
 RUN cd /tmp && \
@@ -21,7 +21,7 @@ RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedf
 ARG BRANCH=${BRANCH:-master}
 RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
  && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"


@@ -5,7 +5,7 @@ RUN mkdir -p /go/src/github.com/seaweedfs/
 ADD . /go/src/github.com/seaweedfs/seaweedfs
 RUN ls -al /go/src/github.com/seaweedfs/ && \
  cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
  && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"


@@ -7,7 +7,7 @@ gen: dev
 cgo ?= 0
 binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
-	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
+	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(SWCOMMIT)"
 	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
 	cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/
 	cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/
@@ -56,7 +56,7 @@ require (
 	github.com/klauspost/reedsolomon v1.12.4
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.9.9
+	github.com/linxGnu/grocksdb v1.10.1
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -94,7 +94,7 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.0
+	go.etcd.io/etcd/client/v3 v3.6.1
 	go.mongodb.org/mongo-driver v1.17.3
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.41.0
@@ -117,7 +117,7 @@ require (
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/mathutil v1.7.1
 	modernc.org/memory v1.11.0 // indirect
-	modernc.org/sqlite v1.37.1
+	modernc.org/sqlite v1.38.0
 	modernc.org/strutil v1.2.1
 )

@@ -127,14 +127,14 @@ require (
 	github.com/armon/go-metrics v0.4.1
 	github.com/aws/aws-sdk-go-v2 v1.36.3
 	github.com/aws/aws-sdk-go-v2/config v1.29.14
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.68
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.0
 	github.com/getsentry/sentry-go v0.31.1
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.7.2
+	github.com/hanwen/go-fuse/v2 v2.7.3-0.20250605191109-50f6569d1a7d
 	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
 	github.com/minio/crc64nvme v1.0.1
@@ -144,14 +144,14 @@ require (
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rclone/rclone v1.69.3
 	github.com/rdleal/intervalst v1.4.1
-	github.com/redis/go-redis/v9 v9.8.0
+	github.com/redis/go-redis/v9 v9.10.0
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/tarantool/go-tarantool/v2 v2.3.2
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.108.3
-	go.etcd.io/etcd/client/pkg/v3 v3.6.0
+	go.etcd.io/etcd/client/pkg/v3 v3.6.1
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.14.0
 	google.golang.org/grpc/security/advancedtls v1.0.0
@@ -205,7 +205,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 // indirect
 	github.com/aws/smithy-go v1.22.3 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.2 // indirect
@@ -213,7 +213,7 @@ require (
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
 	github.com/cloudinary/cloudinary-go/v2 v2.9.0 // indirect
 	github.com/cloudsoda/go-smb2 v0.0.0-20241223203758-52b943b88fd6 // indirect
 	github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
@@ -346,7 +346,7 @@ require (
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.3.10 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
@@ -366,7 +366,7 @@ require (
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/libc v1.65.7 // indirect
+	modernc.org/libc v1.65.10 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	storj.io/common v0.0.0-20240812101423-26b53789c348 // indirect
 	storj.io/drpc v0.0.35-0.20240709171858-0075ac871661 // indirect
@@ -659,8 +659,8 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
 github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
 github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.68 h1:cFb9yjI02/sWHBSYXAtkamjzCuRymvmeFmt0TC0MbYY=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.68/go.mod h1:H6E+jBzyqUu8u0vGaU6POkK3P0NylYEeRZ6ynBpMqIk=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 h1:6VFPH/Zi9xYFMJKPQOX5URYkQoXRWeJ7V/7Y6ZDYoms=
@@ -681,8 +681,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0 h1:fV4XIU5sn/x8gjRouoJpDVHj+ExJaUk4prYF+eb6qTs=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1 h1:xYEAf/6QHiTZDccKnPMbsMwlau13GsDsTgdue3wmHGw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE=
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.2 h1:PajtbJ/5bEo6iUAIGMYnK8ljqg2F1h4mMCGh1acjN30=
 github.com/aws/aws-sdk-go-v2/service/sns v1.34.2/go.mod h1:PJtxxMdj747j8DeZENRTTYAz/lx/pADn/U0k7YNNiUY=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.3 h1:j5BchjfDoS7K26vPdyJlyxBIIBGDflq3qjjJKBDlbcI=
@@ -691,8 +691,8 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 h1:oIaQ1e17CSKaWmUTu62MtraRWVIosn/iONMuZt0gbqc=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.20/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
 github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
 github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -750,8 +750,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
-github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
-github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
+github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
 github.com/cloudinary/cloudinary-go/v2 v2.9.0 h1:8C76QklmuV4qmKAC7cUnu9D68X9kCkFMuLspPikECCo=
 github.com/cloudinary/cloudinary-go/v2 v2.9.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
 github.com/cloudsoda/go-smb2 v0.0.0-20241223203758-52b943b88fd6 h1:mLY/79N73URZ2J/oRKTxmfhCgxThzBmjQ6XOjX5tYjI=
@@ -1125,8 +1125,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5uk
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
-github.com/hanwen/go-fuse/v2 v2.7.2 h1:SbJP1sUP+n1UF8NXBA14BuojmTez+mDgOk0bC057HQw=
-github.com/hanwen/go-fuse/v2 v2.7.2/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48=
+github.com/hanwen/go-fuse/v2 v2.7.3-0.20250605191109-50f6569d1a7d h1:XuIObrEyEA4ujlzIkIASZuI1CD7yGuxZrzo3vc2DQhk=
+github.com/hanwen/go-fuse/v2 v2.7.3-0.20250605191109-50f6569d1a7d/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -1267,8 +1267,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/linxGnu/grocksdb v1.9.9 h1:CzSS/vHLtVIdxdrjvqWR/sm93u/0eB6UcoO14YqObnw=
-github.com/linxGnu/grocksdb v1.9.9/go.mod h1:C3CNe9UYc9hlEM2pC82AqiGS3LRW537u9LFV4wIZuHk=
+github.com/linxGnu/grocksdb v1.10.1 h1:YX6gUcKvSC3d0s9DaqgbU+CRkZHzlELgHu1Z/kmtslg=
+github.com/linxGnu/grocksdb v1.10.1/go.mod h1:C3CNe9UYc9hlEM2pC82AqiGS3LRW537u9LFV4wIZuHk=
 github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
 github.com/lpar/date v1.0.0/go.mod h1:KjYe0dDyMQTgpqcUz4LEIeM5VZwhggjVx/V2dtc8NSo=
 github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0=
@@ -1459,8 +1459,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.4.1 h1:8rCuAea0aj3Z6cDSeN3Q3c+LZjmHmrqZhXrJbqfPw0c=
 github.com/rdleal/intervalst v1.4.1/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
-github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
+github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs=
+github.com/redis/go-redis/v9 v9.10.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo=
 github.com/redis/rueidis v1.0.19/go.mod h1:8B+r5wdnjwK3lTFml5VtxjzGOQAC+5UmujoD12pDrEo=
 github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
@@ -1674,12 +1674,12 @@ go.einride.tech/aip v0.68.1 h1:16/AfSxcQISGN5z9C5lM+0mLYXihrHbQ1onvYTr93aQ=
 go.einride.tech/aip v0.68.1/go.mod h1:XaFtaj4HuA3Zwk9xoBtTWgNubZ0ZZXv9BZJCkuKuWbg=
 go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
 go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
-go.etcd.io/etcd/api/v3 v3.6.0 h1:vdbkcUBGLf1vfopoGE/uS3Nv0KPyIpUV/HM6w9yx2kM=
-go.etcd.io/etcd/api/v3 v3.6.0/go.mod h1:Wt5yZqEmxgTNJGHob7mTVBJDZNXiHPtXTcPab37iFOw=
-go.etcd.io/etcd/client/pkg/v3 v3.6.0 h1:nchnPqpuxvv3UuGGHaz0DQKYi5EIW5wOYsgUNRc365k=
-go.etcd.io/etcd/client/pkg/v3 v3.6.0/go.mod h1:Jv5SFWMnGvIBn8o3OaBq/PnT0jjsX8iNokAUessNjoA=
-go.etcd.io/etcd/client/v3 v3.6.0 h1:/yjKzD+HW5v/3DVj9tpwFxzNbu8hjcKID183ug9duWk=
-go.etcd.io/etcd/client/v3 v3.6.0/go.mod h1:Jzk/Knqe06pkOZPHXsQ0+vNDvMQrgIqJ0W8DwPdMJMg=
+go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo=
+go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto=
+go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc=
+go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
+go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
 go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ=
 go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2512,8 +2512,8 @@ modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJD
 modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
 modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
 modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
-modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8=
-modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
+modernc.org/fileutil v1.3.3 h1:3qaU+7f7xxTUmvU1pJTZiDLAIoJVdUSSauJNHg9yXoA=
+modernc.org/fileutil v1.3.3/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
 modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
 modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
 modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
@@ -2524,8 +2524,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
 modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
 modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
 modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
-modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00=
-modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU=
+modernc.org/libc v1.65.10 h1:ZwEk8+jhW7qBjHIT+wd0d9VjitRyQef9BnzlzGwMODc=
+modernc.org/libc v1.65.10/go.mod h1:StFvYpx7i/mXtBAfVOjaU0PWZOvIRoZSgXhrwXzr8Po=
 modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
@@ -2544,8 +2544,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
 modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
 modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
 modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
-modernc.org/sqlite v1.37.1 h1:EgHJK/FPoqC+q2YBXg7fUmES37pCHFc97sI7zSayBEs=
-modernc.org/sqlite v1.37.1/go.mod h1:XwdRtsE1MpiBcL54+MbKcaDvcuej+IYSMfLN6gSKV8g=
+modernc.org/sqlite v1.38.0 h1:+4OrfPQ8pxHKuWG4md1JpR/EYAh3Md7TdejuuzE7EUI=
+modernc.org/sqlite v1.38.0/go.mod h1:1Bj+yES4SVvBZ4cBOpVZ6QgesMCKpJZDq0nxYzOpmNE=
 modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
 modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.89"
+appVersion: "3.91"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.389
+version: 4.0.391
@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io"
 	"math"
 	"math/rand"
@@ -115,7 +116,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 	util.LoadSecurityConfiguration()
 	b.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")

-	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
+	fmt.Printf("This is SeaweedFS version %s %s %s\n", version.Version(), runtime.GOOS, runtime.GOARCH)
 	if *b.maxCpu < 1 {
 		*b.maxCpu = runtime.NumCPU()
 	}
@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net"
 	"net/http"
 	"os"
@@ -329,7 +330,7 @@ func (fo *FilerOptions) startFiler() {

 	if *fo.publicPort != 0 {
 		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
-		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
+		glog.V(0).Infoln("Start Seaweed filer server", version.Version(), "public at", publicListeningAddress)
 		publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
 		if e != nil {
 			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
@@ -348,7 +349,7 @@ func (fo *FilerOptions) startFiler() {
 		}
 	}

-	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
+	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", version.Version(), *fo.ip, *fo.port)
 	filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
 		*fo.bindIp, *fo.port,
 		time.Duration(10)*time.Second,
@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"

 	"time"
@@ -89,7 +90,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 		glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
 	}

-	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
+	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", version.Version(), *iamopt.port)
 	if iamApiLocalListener != nil {
 		go func() {
 			if err = httpS.Serve(iamApiLocalListener); err != nil {
@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"os"
 	"path"
@@ -160,7 +161,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
 	listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
-	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
+	glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
 	masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
@@ -211,7 +212,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		protobuf.RegisterRaftServer(grpcS, raftServer)
 	}
 	reflection.Register(grpcS)
-	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
+	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", version.Version(), *masterOption.ipBind, grpcPort)
 	if grpcLocalL != nil {
 		go grpcS.Serve(grpcLocalL)
 	}
@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"time"

@@ -119,7 +120,7 @@ func startMasterFollower(masterOptions MasterOptions) {
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, option, masters)
 	listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port)
-	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
+	glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
 	masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
@@ -134,7 +135,7 @@ func startMasterFollower(masterOptions MasterOptions) {
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
 	reflection.Register(grpcS)
-	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort)
+	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", version.Version(), *masterOptions.ip, grpcPort)
 	if grpcLocalL != nil {
 		go grpcS.Serve(grpcLocalL)
 	}
@@ -0,0 +1,5 @@
+package command
+
+func checkMountPointAvailable(dir string) bool {
+	return true
+}
@@ -1,5 +1,5 @@
-//go:build !linux && !darwin
-// +build !linux,!darwin
+//go:build !linux && !darwin && !freebsd
+// +build !linux,!darwin,!freebsd

 package command

@@ -1,5 +1,5 @@
-//go:build linux || darwin
-// +build linux darwin
+//go:build linux || darwin || freebsd
+// +build linux darwin freebsd

 package command

@@ -16,6 +16,8 @@ import (
 	"syscall"
 	"time"

+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+
 	"github.com/hanwen/go-fuse/v2/fuse"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mount"
@@ -290,7 +292,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	}

 	glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
-	glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
+	glog.V(0).Infof("This is SeaweedFS version %s %s %s", version.Version(), runtime.GOOS, runtime.GOARCH)

 	server.Serve()

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -339,7 +340,7 @@ func (s3opt *S3Options) startS3Server() bool {
 			glog.Fatalf("error with tls config: %v", err)
 		}
 		if *s3opt.portHttps == 0 {
-			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
+			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.port)
 			if s3ApiLocalListener != nil {
 				go func() {
 					if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
@@ -351,9 +352,9 @@ func (s3opt *S3Options) startS3Server() bool {
 				glog.Fatalf("S3 API Server Fail to serve: %v", err)
 			}
 		} else {
-			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.portHttps)
+			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.portHttps)
 			s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
-                *s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
+				*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
 			if s3ApiLocalListenerHttps != nil {
 				go func() {
 					if err = httpS.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
@@ -369,7 +370,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		}
 	}
 	if *s3opt.tlsPrivateKey == "" || *s3opt.portHttps > 0 {
-		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", version.Version(), *s3opt.port)
 		if s3ApiLocalListener != nil {
 			go func() {
 				if err = httpS.Serve(s3ApiLocalListener); err != nil {
@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net"
 	"os"
 	"runtime"
@@ -175,7 +176,7 @@ func (sftpOpt *SftpOptions) startSftpServer() bool {
 		glog.Fatalf("SFTP server listener on %s error: %v", listenAddress, err)
 	}

-	glog.V(0).Infof("Start Seaweed SFTP Server %s at %s", util.Version(), listenAddress)
+	glog.V(0).Infof("Start Seaweed SFTP Server %s at %s", version.Version(), listenAddress)

 	if sftpLocalListener != nil {
 		go func() {
@@ -10,6 +10,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	swv "github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io"
 	"net/http"
 	"os"
@@ -117,7 +118,7 @@ func runUpdate(cmd *Command, args []string) bool {
 }

 func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) {
-	currentVersion := util.VERSION_NUMBER
+	currentVersion := swv.VERSION_NUMBER
 	rel, err := GitHubLatestRelease(ctx, ver, "seaweedfs", "seaweedfs")
 	if err != nil {
 		return "", err
@@ -2,9 +2,8 @@ package command

 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"runtime"
-
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 var cmdVersion = &Command{
@@ -19,6 +18,6 @@ func runVersion(cmd *Command, args []string) bool {
 		cmd.Usage()
 	}

-	fmt.Printf("version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
+	fmt.Printf("version %s %s %s\n", version.Version(), runtime.GOOS, runtime.GOARCH)
 	return true
 }
@@ -2,6 +2,7 @@ package command

 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
@@ -351,7 +352,7 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe

 func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
 	publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort)
-	glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
+	glog.V(0).Infoln("Start Seaweed volume server", version.Version(), "public at", publicListeningAddress)
 	publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)
@@ -378,7 +379,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd
 	}

 	listeningAddress := util.JoinHostPort(*v.bindIp, *v.port)
-	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
+	glog.V(0).Infof("Start Seaweed volume server %s at %s", version.Version(), listeningAddress)
 	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)
@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"os"
 	"os/user"
@@ -65,7 +66,7 @@ func runWebDav(cmd *Command, args []string) bool {
 	util.LoadSecurityConfiguration()

 	listenAddress := fmt.Sprintf("%s:%d", *webDavStandaloneOptions.ipBind, *webDavStandaloneOptions.port)
-	glog.V(0).Infof("Starting Seaweed WebDav Server %s at %s", util.Version(), listenAddress)
+	glog.V(0).Infof("Starting Seaweed WebDav Server %s at %s", version.Version(), listenAddress)

 	return webDavStandaloneOptions.startWebDav()

@@ -136,12 +137,12 @@ func (wo *WebDavOption) startWebDav() bool {
 	}

 	if *wo.tlsPrivateKey != "" {
-		glog.V(0).Infof("Start Seaweed WebDav Server %s at https %s", util.Version(), listenAddress)
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at https %s", version.Version(), listenAddress)
 		if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
 			glog.Fatalf("WebDav Server Fail to serve: %v", err)
 		}
 	} else {
-		glog.V(0).Infof("Start Seaweed WebDav Server %s at http %s", util.Version(), listenAddress)
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at http %s", version.Version(), listenAddress)
 		if err = httpS.Serve(webDavListener); err != nil {
 			glog.Fatalf("WebDav Server Fail to serve: %v", err)
 		}
@@ -169,7 +169,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we

 	iter := store.db.NewIterator(ro)
 	defer iter.Close()
-	err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool {
+	err = enumerate(iter, directoryPrefix, nil, false, -1, "", func(key, value []byte) bool {
 		batch.Delete(key)
 		return true
 	})
@@ -186,23 +186,16 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
 	return nil
 }

-func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int64, fn func(key, value []byte) bool) (err error) {
+func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int64, startFileName string, fn func(key, value []byte) bool) (err error) {

 	if len(lastKey) == 0 {
 		iter.Seek(prefix)
 	} else {
 		iter.Seek(lastKey)
-		if !includeLastKey {
-			if iter.Valid() {
-				if bytes.Equal(iter.Key().Data(), lastKey) {
-					iter.Next()
-				}
-			}
-		}
 	}

 	i := int64(0)
-	for ; iter.Valid(); iter.Next() {
+	for iter.Valid() {

 		if limit > 0 {
 			i++
@@ -217,12 +210,23 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey
 			break
 		}

+		fileName := getNameFromKey(key)
+		if fileName == "" {
+			iter.Next()
+			continue
+		}
+		if fileName == startFileName && !includeLastKey {
+			iter.Next()
+			continue
+		}
+
 		ret := fn(key, iter.Value().Data())

 		if !ret {
 			break
 		}

+		iter.Next()
 	}

 	if err := iter.Err(); err != nil {
@@ -249,7 +253,7 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir

 	iter := store.db.NewIterator(ro)
 	defer iter.Close()
-	err = enumerate(iter, directoryPrefix, lastFileStart, includeStartFile, limit, func(key, value []byte) bool {
+	err = enumerate(iter, directoryPrefix, lastFileStart, includeStartFile, limit, startFileName, func(key, value []byte) bool {
 		fileName := getNameFromKey(key)
 		if fileName == "" {
 			return true
@@ -11,6 +11,7 @@ import (
 	"time"

 	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )

@@ -112,3 +113,73 @@ func BenchmarkInsertEntry(b *testing.B) {
 		store.InsertEntry(ctx, entry)
 	}
 }
+
+func TestListDirectoryWithPrefix(t *testing.T) {
+	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+	dir := t.TempDir()
+	store := &RocksDBStore{}
+	store.initialize(dir)
+	testFiler.SetStore(store)
+
+	ctx := context.Background()
+
+	files := []string{
+		"/bucket1/test-prefix1/file1.txt",
+		"/bucket1/test-prefix1/file2.txt",
+		"/bucket1/test-prefix1-extra.txt",
+	}
+
+	expected1 := []string{
+		"/bucket1/test-prefix1",
+		"/bucket1/test-prefix1-extra.txt",
+	}
+
+	expected2 := []string{
+		"/bucket1/test-prefix1/file1.txt",
+		"/bucket1/test-prefix1/file2.txt",
+	}
+
+	for _, file := range files {
+		fullpath := util.FullPath(file)
+		entry := &filer.Entry{
+			FullPath: fullpath,
+			Attr: filer.Attr{
+				Mode: 0644,
+				Uid:  1,
+				Gid:  1,
+			},
+		}
+		if err := testFiler.CreateEntry(ctx, entry, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
+			t.Fatalf("Failed to create entry %s: %v", fullpath, err)
+		}
+	}
+
+	prefix1 := "test-prefix1"
+	entries1, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/bucket1"), "", false, 100, prefix1, "", "")
+	if err != nil {
+		t.Fatalf("Failed to list entries with prefix %s: %v", prefix1, err)
+	}
+	if len(entries1) != 2 {
+		t.Errorf("Expected 2 entries with prefix %s, got %d", prefix1, len(entries1))
+	} else {
+		t.Logf("Found %d entries with prefix %s", len(entries1), prefix1)
+	}
+	for i, entry := range entries1 {
+		if string(entry.FullPath) != expected1[i] {
+			t.Errorf("Expected entry %s, got %s", expected1[i], entry.FullPath)
+		}
+	}
+
+	entries2, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/bucket1/test-prefix1"), "", false, 100, "", "", "")
+	if err != nil {
+		t.Fatalf("Failed to list entries with prefix %s: %v", prefix1, err)
+	}
+	if len(entries2) != 2 {
+		t.Errorf("Expected 2 entries with prefix %s, got %d", prefix1, len(entries1))
+	}
+	for i, entry := range entries2 {
+		if string(entry.FullPath) != expected2[i] {
+			t.Errorf("Expected entry %s, got %s", expected2[i], entry.FullPath)
+		}
+	}
+}
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sync"
 	"sync/atomic"
 	"time"

@@ -22,6 +23,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"

 	"github.com/hanwen/go-fuse/v2/fs"
@@ -70,19 +72,21 @@ type WFS struct {
 	fuse.RawFileSystem
 	mount_pb.UnimplementedSeaweedMountServer
 	fs.Inode
-	option            *Option
-	metaCache         *meta_cache.MetaCache
-	stats             statsCache
-	chunkCache        *chunk_cache.TieredChunkCache
-	signature         int32
-	concurrentWriters *util.LimitedConcurrentExecutor
-	inodeToPath       *InodeToPath
-	fhMap             *FileHandleToInode
-	dhMap             *DirectoryHandleToInode
-	fuseServer        *fuse.Server
-	IsOverQuota       bool
-	fhLockTable       *util.LockTable[FileHandleId]
-	FilerConf         *filer.FilerConf
+	option               *Option
+	metaCache            *meta_cache.MetaCache
+	stats                statsCache
+	chunkCache           *chunk_cache.TieredChunkCache
+	signature            int32
+	concurrentWriters    *util.LimitedConcurrentExecutor
+	copyBufferPool       sync.Pool
+	concurrentCopiersSem chan struct{}
+	inodeToPath          *InodeToPath
+	fhMap                *FileHandleToInode
+	dhMap                *DirectoryHandleToInode
+	fuseServer           *fuse.Server
+	IsOverQuota          bool
+	fhLockTable          *util.LockTable[FileHandleId]
+	FilerConf            *filer.FilerConf
 }

 func NewSeaweedFileSystem(option *Option) *WFS {
@@ -138,6 +142,10 @@ func NewSeaweedFileSystem(option *Option) *WFS {

 	if wfs.option.ConcurrentWriters > 0 {
 		wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters)
+		wfs.concurrentCopiersSem = make(chan struct{}, wfs.option.ConcurrentWriters)
+	}
+	wfs.copyBufferPool.New = func() any {
+		return make([]byte, option.ChunkSizeLimit)
 	}
 	return wfs
 }
@@ -182,7 +190,6 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle
 }

 func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) {
-
 	// glog.V(3).Infof("read entry cache miss %s", fullpath)
 	dir, name := fullpath.DirAndName()

@@ -231,7 +238,7 @@ func (wfs *WFS) ClearCacheDir() {
 }

 func (option *Option) setupUniqueCacheDirectory() {
-	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8]
+	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + version.Version()))[0:8]
 	option.uniqueCacheDirForRead = path.Join(option.CacheDirForRead, cacheUniqueId)
 	os.MkdirAll(option.uniqueCacheDirForRead, os.FileMode(0777)&^option.Umask)
 	option.uniqueCacheDirForWrite = filepath.Join(path.Join(option.CacheDirForWrite, cacheUniqueId), "swap")
@@ -0,0 +1,8 @@
+package mount
+
+import (
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+}
@@ -1,13 +1,13 @@
 package mount

 import (
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"net/http"
 	"time"

 	"github.com/hanwen/go-fuse/v2/fuse"

 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 // CopyFileRange copies data from one file to another from and to specified offsets.
@@ -70,30 +70,85 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
 		in.OffOut, in.OffOut+in.Len,
 	)

-	data := make([]byte, in.Len)
-	totalRead, err := readDataByFileHandle(data, fhIn, int64(in.OffIn))
-	if err != nil {
-		glog.Warningf("file handle read %s %d: %v", fhIn.FullPath(), totalRead, err)
-		return 0, fuse.EIO
+	// Concurrent copy operations could allocate too much memory, so we want to
+	// throttle our concurrency, scaling with the number of writers the mount
+	// was configured with.
+	if wfs.concurrentCopiersSem != nil {
+		wfs.concurrentCopiersSem <- struct{}{}
+		defer func() { <-wfs.concurrentCopiersSem }()
 	}
-	data = data[:totalRead]

-	if totalRead == 0 {
+	// We want to stream the copy operation to avoid allocating massive buffers.
+	nowUnixNano := time.Now().UnixNano()
+	totalCopied := int64(0)
+	buff := wfs.copyBufferPool.Get().([]byte)
+	defer wfs.copyBufferPool.Put(buff)
+	for {
+		// Comply with cancellation as best as we can, given that the underlying
+		// IO functions aren't cancellation-aware.
+		select {
+		case <-cancel:
+			glog.Warningf("canceled CopyFileRange for %s (copied %d)",
+				fhIn.FullPath(), totalCopied)
+			return uint32(totalCopied), fuse.EINTR
+		default: // keep going
+		}
+
+		// We can save one IO by breaking early if we already know the next read
+		// will result in zero bytes.
+		remaining := int64(in.Len) - totalCopied
+		readLen := min(remaining, int64(len(buff)))
+		if readLen == 0 {
+			break
+		}
+
+		// Perform the read
+		offsetIn := totalCopied + int64(in.OffIn)
+		numBytesRead, err := readDataByFileHandle(
+			buff[:readLen], fhIn, offsetIn)
+		if err != nil {
+			glog.Warningf("file handle read %s %d (total %d): %v",
+				fhIn.FullPath(), numBytesRead, totalCopied, err)
+			return 0, fuse.EIO
+		}
+
+		// Break if we're done copying (no more bytes to read)
+		if numBytesRead == 0 {
+			break
+		}
+
+		offsetOut := int64(in.OffOut) + totalCopied
+
+		// Detect mime type only during the beginning of our stream, since
+		// DetectContentType is expecting some of the first 512 bytes of the
+		// file. See [http.DetectContentType] for details.
+		if offsetOut <= 512 {
+			fhOut.contentType = http.DetectContentType(buff[:numBytesRead])
+		}
+
+		// Perform the write
+		fhOut.dirtyPages.writerPattern.MonitorWriteAt(offsetOut, int(numBytesRead))
+		fhOut.dirtyPages.AddPage(
+			offsetOut,
+			buff[:numBytesRead],
+			fhOut.dirtyPages.writerPattern.IsSequentialMode(),
+			nowUnixNano)
+
+		// Accumulate for the next loop iteration
+		totalCopied += numBytesRead
+	}
+
+	if totalCopied == 0 {
 		return 0, fuse.OK
 	}

-	// put data at the specified offset in target file
-	fhOut.dirtyPages.writerPattern.MonitorWriteAt(int64(in.OffOut), int(in.Len))
+	fhOut.entry.Attributes.FileSize = uint64(max(
+		totalCopied+int64(in.OffOut),
+		int64(fhOut.entry.Attributes.FileSize),
+	))
 	fhOut.entry.Content = nil
-	fhOut.dirtyPages.AddPage(int64(in.OffOut), data, fhOut.dirtyPages.writerPattern.IsSequentialMode(), time.Now().UnixNano())
-	fhOut.entry.Attributes.FileSize = uint64(max(int64(in.OffOut)+totalRead, int64(fhOut.entry.Attributes.FileSize)))
 	fhOut.dirtyMetadata = true
-	written = uint32(totalRead)
-
-	// detect mime type
-	if written > 0 && in.OffOut <= 512 {
-		fhOut.contentType = http.DetectContentType(data)
-	}

+	written = uint32(totalCopied)
 	return written, fuse.OK
 }
@@ -1,3 +1,6 @@
+//go:build !freebsd
+// +build !freebsd
+
 package mount

 import (
@@ -0,0 +1,27 @@
+package mount
+
+import (
+	"syscall"
+
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+func (wfs *WFS) GetXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string, dest []byte) (size uint32, code fuse.Status) {
+
+	return 0, fuse.Status(syscall.ENOTSUP)
+}
+
+func (wfs *WFS) SetXAttr(cancel <-chan struct{}, input *fuse.SetXAttrIn, attr string, data []byte) fuse.Status {
+
+	return fuse.Status(syscall.ENOTSUP)
+}
+
+func (wfs *WFS) ListXAttr(cancel <-chan struct{}, header *fuse.InHeader, dest []byte) (n uint32, code fuse.Status) {
+
+	return 0, fuse.Status(syscall.ENOTSUP)
+}
+
+func (wfs *WFS) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string) fuse.Status {
+
+	return fuse.Status(syscall.ENOTSUP)
+}
@@ -2,6 +2,7 @@ package s3

 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io"
 	"reflect"

@@ -53,7 +54,7 @@ func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.R
 		sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
 	}
 	sess.Handlers.Build.PushBack(func(r *request.Request) {
-		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+util.VERSION_NUMBER)
+		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+version.VERSION_NUMBER)
 	})
 	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
 	client.conn = s3.New(sess)
@@ -1,13 +1,13 @@
 package s3api

 import (
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"strconv"
 	"time"

 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 func track(f http.HandlerFunc, action string) http.HandlerFunc {
@@ -17,7 +17,7 @@ func track(f http.HandlerFunc, action string) http.HandlerFunc {
 		defer inFlightGauge.Dec()

 		bucket, _ := s3_constants.GetBucketAndObject(r)
-		w.Header().Set("Server", "SeaweedFS "+util.VERSION)
+		w.Header().Set("Server", "SeaweedFS "+version.VERSION)
 		recorder := stats_collect.NewStatusResponseWriter(w)
 		start := time.Now()
 		f(recorder, r)
@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/google/uuid"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"google.golang.org/grpc/metadata"
 	"io"
 	"io/fs"
@@ -240,19 +241,19 @@ func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly b

 func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]interface{})
-	m["Version"] = util.Version()
+	m["Version"] = version.Version()
 	writeJsonQuiet(w, r, http.StatusOK, m)
 }
 func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]interface{})
-	m["Version"] = util.Version()
+	m["Version"] = version.Version()
 	m["Counters"] = serverStats
 	writeJsonQuiet(w, r, http.StatusOK, m)
 }

 func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]interface{})
-	m["Version"] = util.Version()
+	m["Version"] = version.Version()
 	m["Memory"] = stats.MemStat()
 	writeJsonQuiet(w, r, http.StatusOK, m)
 }
@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"time"

 	"github.com/seaweedfs/seaweedfs/weed/cluster"
@@ -11,7 +12,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) {
@@ -94,10 +94,10 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.
 		Signature:          fs.filer.Signature,
 		MetricsAddress:     fs.metricsAddress,
 		MetricsIntervalSec: int32(fs.metricsIntervalSec),
-		Version:            util.Version(),
+		Version:            version.Version(),
 		FilerGroup:         fs.option.FilerGroup,
-		MajorVersion:       util.MAJOR_VERSION,
-		MinorVersion:       util.MINOR_VERSION,
+		MajorVersion:       version.MAJOR_VERSION,
+		MinorVersion:       version.MINOR_VERSION,
 	}

 	glog.V(4).Infof("GetFilerConfiguration: %v", t)
@@ -30,6 +30,7 @@ func (fs *FilerServer) KvPut(ctx context.Context, req *filer_pb.KvPutRequest) (*
 		if err := fs.filer.Store.KvDelete(ctx, req.Key); err != nil {
 			return &filer_pb.KvPutResponse{Error: err.Error()}, nil
 		}
+		return &filer_pb.KvPutResponse{}, nil
 	}

 	err := fs.filer.Store.KvPut(ctx, req.Key, req.Value)
@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"errors"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"os"
 	"strconv"
@@ -14,8 +15,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 )

@@ -80,7 +79,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}

-	w.Header().Set("Server", "SeaweedFS "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS "+version.VERSION)

 	switch r.Method {
 	case http.MethodGet, http.MethodHead:
@@ -168,7 +167,7 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}

-	w.Header().Set("Server", "SeaweedFS "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS "+version.VERSION)

 	switch r.Method {
 	case http.MethodGet, http.MethodHead:
@@ -229,7 +228,7 @@ func (fs *FilerServer) maybeCheckJwtAuthorization(r *http.Request, isWrite bool)
 }

 func (fs *FilerServer) filerHealthzHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Server", "SeaweedFS "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS "+version.VERSION)
 	if _, err := fs.filer.Store.FindEntry(context.Background(), filer.TopicsDir); err != nil && err != filer_pb.ErrNotFound {
 		glog.Warningf("filerHealthzHandler FindEntry: %+v", err)
 		w.WriteHeader(http.StatusServiceUnavailable)
@@ -2,6 +2,7 @@ package weed_server

 import (
 	"errors"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"strconv"
 	"strings"
@@ -69,7 +70,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
 			ShouldDisplayLoadMore bool
 			EmptyFolder           bool
 		}{
-			util.Version(),
+			version.Version(),
 			path,
 			entries,
 			limit,
@@ -91,7 +92,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
 		EmptyFolder           bool
 		ShowDirectoryDelete   bool
 	}{
-		util.Version(),
+		version.Version(),
 		path,
 		ui.ToBreadcrumb(path),
 		entries,
@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io"
 	"net/http"
 	"os"
@@ -51,7 +52,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 		if err.Error() == "operation not permitted" {
 			writeJsonError(w, r, http.StatusForbidden, err)
 		} else if strings.HasPrefix(err.Error(), "read input:") || err.Error() == io.ErrUnexpectedEOF.Error() {
-			writeJsonError(w, r, util.HttpStatusCancelled, err)
+			writeJsonError(w, r, version.HttpStatusCancelled, err)
 		} else if strings.HasSuffix(err.Error(), "is a file") || strings.HasSuffix(err.Error(), "already exists") {
 			writeJsonError(w, r, http.StatusConflict, err)
 		} else {
@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"math/rand/v2"
 	"net/http"
 	"strconv"
@@ -18,7 +19,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"github.com/seaweedfs/seaweedfs/weed/topology"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )

@@ -49,7 +49,7 @@ func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.R

 func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]interface{})
-	m["Version"] = util.Version()
+	m["Version"] = version.Version()
 	m["Topology"] = ms.Topo.ToInfo()
 	writeJsonQuiet(w, r, http.StatusOK, m)
 }
@@ -104,7 +104,7 @@ func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request

 func (ms *MasterServer) volumeStatusHandler(w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]interface{})
-	m["Version"] = util.Version()
+	m["Version"] = version.Version()
 	m["Volumes"] = ms.Topo.ToVolumeMap()
 	writeJsonQuiet(w, r, http.StatusOK, m)
 }
@@ -204,7 +204,7 @@ func (ms *MasterServer) collectionInfoHandler(w http.ResponseWriter, r *http.Req
 		for i, volumeLayout := range volumeLayouts {
 			volumeLayoutStats := volumeLayout.Stats()
 			m := make(map[string]interface{})
-			m["Version"] = util.Version()
+			m["Version"] = version.Version()
 			m["Collection"] = collectionName
 			m["TotalSize"] = volumeLayoutStats.TotalSize
 			m["FileCount"] = volumeLayoutStats.FileCount
@@ -216,7 +216,7 @@ func (ms *MasterServer) collectionInfoHandler(w http.ResponseWriter, r *http.Req
 	} else {
 		//prepare the json response
 		collectionStats := map[string]interface{}{
-			"Version":     util.Version(),
+			"Version":     version.Version(),
 			"Collection":  collectionName,
 			"TotalSize":   uint64(0),
 			"FileCount":   uint64(0),
@@ -1,6 +1,7 @@
 package weed_server

 import (
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"time"

@@ -9,7 +10,6 @@ import (

 	ui "github.com/seaweedfs/seaweedfs/weed/server/master_ui"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 func (ms *MasterServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) {
@@ -29,7 +29,7 @@ func (ms *MasterServer) uiStatusHandler(w http.ResponseWriter, r *http.Request)
 			Counters          *stats.ServerStats
 			VolumeSizeLimitMB uint32
 		}{
-			util.Version(),
+			version.Version(),
 			ms.Topo.ToInfo(),
 			ms.Topo.RaftServer,
 			infos,
@@ -46,7 +46,7 @@ func (ms *MasterServer) uiStatusHandler(w http.ResponseWriter, r *http.Request)
 			Counters          *stats.ServerStats
 			VolumeSizeLimitMB uint32
 		}{
-			util.Version(),
+			version.Version(),
 			ms.Topo.ToInfo(),
 			ms.Topo.HashicorpRaft,
 			infos,
@@ -3,18 +3,17 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"path/filepath"
 	"time"

 	"github.com/seaweedfs/seaweedfs/weed/storage"

 	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@@ -253,7 +252,7 @@ func (vs *VolumeServer) VolumeServerStatus(ctx context.Context, req *volume_serv

 	resp := &volume_server_pb.VolumeServerStatusResponse{
 		MemoryStatus: stats.MemStat(),
-		Version:      util.Version(),
+		Version:      version.Version(),
 		DataCenter:   vs.dataCenter,
 		Rack:         vs.rack,
 	}
@@ -2,14 +2,13 @@ package weed_server

 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"

-	"github.com/seaweedfs/seaweedfs/weed/util"
-
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
@@ -37,7 +36,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque

 	statusRecorder := stats.NewStatusResponseWriter(w)
 	w = statusRecorder
-	w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS Volume "+version.VERSION)
 	if r.Header.Get("Origin") != "" {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 		w.Header().Set("Access-Control-Allow-Credentials", "true")
@@ -57,7 +56,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
 			select {
 			case <-r.Context().Done():
 				glog.V(4).Infof("request cancelled from %s: %v", r.RemoteAddr, r.Context().Err())
-				w.WriteHeader(util.HttpStatusCancelled)
+				w.WriteHeader(version.HttpStatusCancelled)
 				vs.inFlightDownloadDataLimitCond.L.Unlock()
 				return
 			default:
@@ -130,7 +129,7 @@ func getContentLength(r *http.Request) int64 {
 func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) {
 	statusRecorder := stats.NewStatusResponseWriter(w)
 	w = statusRecorder
-	w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS Volume "+version.VERSION)
 	if r.Header.Get("Origin") != "" {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 		w.Header().Set("Access-Control-Allow-Credentials", "true")
@@ -2,16 +2,16 @@ package weed_server

 import (
 	"github.com/seaweedfs/seaweedfs/weed/topology"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"path/filepath"

 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 func (vs *VolumeServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS Volume "+version.VERSION)
 	volumeInfos := vs.store.VolumeInfos()
 	for _, vinfo := range volumeInfos {
 		if len(vinfo.Collection) == 0 {
@@ -29,9 +29,9 @@ func (vs *VolumeServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
 }

 func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS Volume "+version.VERSION)
 	m := make(map[string]interface{})
-	m["Version"] = util.Version()
+	m["Version"] = version.Version()
 	var ds []*volume_server_pb.DiskStatus
 	for _, loc := range vs.store.Locations {
 		if dir, e := filepath.Abs(loc.Directory); e == nil {
@@ -46,9 +46,9 @@ func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {
 }

 func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS Volume "+version.VERSION)
 	m := make(map[string]interface{})
-	m["Version"] = util.Version()
+	m["Version"] = version.Version()
 	var ds []*volume_server_pb.DiskStatus
 	for _, loc := range vs.store.Locations {
 		if dir, e := filepath.Abs(loc.Directory); e == nil {
@@ -2,6 +2,7 @@ package weed_server

 import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"path/filepath"
 	"time"
@@ -10,11 +11,10 @@ import (
 	ui "github.com/seaweedfs/seaweedfs/weed/server/volume_server_ui"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
+	w.Header().Set("Server", "SeaweedFS Volume "+version.VERSION)
 	infos := make(map[string]interface{})
 	infos["Up Time"] = time.Now().Sub(startTime).String()
 	var ds []*volume_server_pb.DiskStatus
@@ -44,7 +44,7 @@ func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request)
 		Stats         interface{}
 		Counters      *stats.ServerStats
 	}{
-		util.Version(),
+		version.Version(),
 		vs.SeedMasterNodes,
 		normalVolumeInfos,
 		vs.store.EcVolumes(),
@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"io"
 	"os"
 	"path"
@@ -129,7 +130,7 @@ type WebDavFile struct {

 func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {

-	cacheUniqueId := util.Md5String([]byte("webdav" + string(option.Filer) + util.Version()))[0:8]
+	cacheUniqueId := util.Md5String([]byte("webdav" + string(option.Filer) + version.Version()))[0:8]
 	cacheDir := path.Join(option.CacheDir, cacheUniqueId)

 	os.MkdirAll(cacheDir, os.FileMode(0755))
@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"io"
 	"time"

@@ -107,7 +108,7 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 		balanceCollections = collectCollectionsForVolumeIds(topologyInfo, volumeIds)
 	} else {
 		// apply to all volumes for the given collection
-		volumeIds, err = collectVolumeIdsForEcEncode(commandEnv, *collection, *fullPercentage, *quietPeriod)
+		volumeIds, err = collectVolumeIdsForEcEncode(commandEnv, *collection, nil, *fullPercentage, *quietPeriod)
 		if err != nil {
 			return err
 		}
@@ -250,7 +251,7 @@ func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId,

 }

-func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) {
+func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection string, sourceDiskType *types.DiskType, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) {
 	// collect topology information
 	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
@@ -270,7 +271,8 @@ func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection stri
 				if v.RemoteStorageName != "" && v.RemoteStorageKey != "" {
 					continue
 				}
-				if v.Collection == selectedCollection && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds {
+				if v.Collection == selectedCollection && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds &&
+					(sourceDiskType == nil || types.ToDiskType(v.DiskType) == *sourceDiskType) {
 					if float64(v.Size) > fullPercentage/100*float64(volumeSizeLimitMb)*1024*1024 {
 						if good, found := vidMap[v.Id]; found {
 							if good {
@@ -9,6 +9,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"path/filepath"
 	"slices"
+	"sort"
 	"strings"
 	"time"

@@ -73,9 +74,34 @@ func (c *commandVolumeList) Do(args []string, commandEnv *CommandEnv, writer io.
 	return nil
 }

+func sortMapKey[T1 comparable, T2 any](m map[T1]T2) []T1 {
+	keys := make([]T1, 0, len(m))
+	for k, _ := range m {
+		keys = append(keys, k)
+	}
+
+	sort.Slice(keys, func(i, j int) bool {
+		switch v1 := any(keys[i]).(type) {
+		case int:
+			return v1 < any(keys[j]).(int)
+		case string:
+			if strings.Compare(v1, any(keys[j]).(string)) < 0 {
+				return true
+			}
+			return false
+		default:
+			return false
+		}
+	})
+	return keys
+}
+
 func diskInfosToString(diskInfos map[string]*master_pb.DiskInfo) string {
 	var buf bytes.Buffer
-	for diskType, diskInfo := range diskInfos {
+
+	for _, diskType := range sortMapKey(diskInfos) {
+		diskInfo := diskInfos[diskType]
+
 		if diskType == "" {
 			diskType = types.HddType
 		}
@@ -152,7 +178,8 @@ func (c *commandVolumeList) writeRackInfo(writer io.Writer, t *master_pb.RackInf
 func (c *commandVolumeList) writeDataNodeInfo(writer io.Writer, t *master_pb.DataNodeInfo, verbosityLevel int, outRackInfo func()) statistics {
 	var s statistics
 	diskInfoFound := false
-	for _, diskInfo := range t.DiskInfos {
+	for _, diskType := range sortMapKey(t.DiskInfos) {
+		diskInfo := t.DiskInfos[diskType]
 		s = s.plus(c.writeDiskInfo(writer, diskInfo, verbosityLevel, func() {
 			outRackInfo()
 			output(verbosityLevel >= 3, writer, "      DataNode %s%s\n", t.Id, diskInfosToString(t.DiskInfos))
@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"io"
 	"time"

@@ -69,6 +70,7 @@ func (c *commandVolumeTierUpload) Do(args []string, commandEnv *CommandEnv, writ
 	quietPeriod := tierCommand.Duration("quietFor", 24*time.Hour, "select volumes without no writes for this period")
 	dest := tierCommand.String("dest", "", "the target tier name")
 	keepLocalDatFile := tierCommand.Bool("keepLocalDatFile", false, "whether keep local dat file")
+	disk := tierCommand.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	if err = tierCommand.Parse(args); err != nil {
 		return nil
 	}
@@ -84,9 +86,15 @@ func (c *commandVolumeTierUpload) Do(args []string, commandEnv *CommandEnv, writ
 		return doVolumeTierUpload(commandEnv, writer, *collection, vid, *dest, *keepLocalDatFile)
 	}

+	var diskType *types.DiskType
+	if disk != nil {
+		_diskType := types.ToDiskType(*disk)
+		diskType = &_diskType
+	}
+
 	// apply to all volumes in the collection
 	// reusing collectVolumeIdsForEcEncode for now
-	volumeIds, err := collectVolumeIdsForEcEncode(commandEnv, *collection, *fullPercentage, *quietPeriod)
+	volumeIds, err := collectVolumeIdsForEcEncode(commandEnv, *collection, diskType, *fullPercentage, *quietPeriod)
 	if err != nil {
 		return err
 	}
@@ -147,11 +155,15 @@ func uploadDatToRemoteTier(grpcDialOption grpc.DialOption, writer io.Writer, vol
 			KeepLocalDatFile:       keepLocalDatFile,
 		})

-		if stream == nil && copyErr == nil {
-			// when the volume is already uploaded, VolumeTierMoveDatToRemote will return nil stream and nil error
-			// so we should directly return in this case
-			fmt.Fprintf(writer, "volume %v already uploaded", volumeId)
-			return nil
+		if stream == nil {
+			if copyErr == nil {
+				// when the volume is already uploaded, VolumeTierMoveDatToRemote will return nil stream and nil error
+				// so we should directly return in this caseAdd commentMore actions
+				fmt.Fprintf(writer, "volume %v already uploaded", volumeId)
+				return nil
+			} else {
+				return copyErr
+			}
 		}
 		var lastProcessed int64
 		for {
@@ -2,13 +2,12 @@ package s3_backend

 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"sync"

-	"github.com/aws/aws-sdk-go/aws/request"
-	"github.com/seaweedfs/seaweedfs/weed/util"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
@@ -51,7 +50,7 @@ func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string,
 		return nil, fmt.Errorf("create aws session in region %s: %v", region, err)
 	}
 	sess.Handlers.Build.PushBack(func(r *request.Request) {
-		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+util.VERSION_NUMBER)
+		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+version.VERSION_NUMBER)
 	})

 	t := s3.New(sess)
@@ -0,0 +1,160 @@
+# Needle Layout Format
+
+This document describes the binary layout of the Needle structure as used in SeaweedFS storage, for all supported versions (v1, v2, v3).
+
+A Needle represents a file or data blob stored in a volume file. The layout determines how the Needle is serialized to disk for efficient storage and retrieval.
+
+---
+
+## Common Field Sizes
+
+| Field         | Size (bytes) |
+|---------------|--------------|
+| Cookie        | 4            |
+| NeedleId      | 8            |
+| Size          | 4            |
+| DataSize      | 4            |
+| Flags         | 1            |
+| NameSize      | 1            |
+| MimeSize      | 1            |
+| LastModified  | 5            |
+| Ttl           | 2            |
+| PairsSize     | 2            |
+| Checksum      | 4            |
+| Timestamp     | 8            |
+
+---
+
+## Needle Layouts by Version
+
+### Version 1
+
+| Offset   | Field    | Size (bytes) | Description                                   |
+|----------|----------|--------------|-----------------------------------------------|
+| 0        | Cookie   | 4            | Random number to mitigate brute force lookups |
+| 4        | Id       | 8            | Needle ID                                     |
+| 12       | Size     | 4            | Length of Data                                |
+| 16       | Data     | N            | File data (N = Size)                          |
+| 16+N     | Checksum | 4            | CRC32 of Data                                 |
+| 20+N     | Padding  | 0-7          | To align to 8 bytes                           |
+
+### Version 2
+
+| Offset   | Field        | Size (bytes) | Description                                   |
+|----------|-------------|--------------|-----------------------------------------------|
+| 0        | Cookie      | 4            | Random number                                 |
+| 4        | Id          | 8            | Needle ID                                     |
+| 12       | Size        | 4            | Total size of the following fields            |
+| 16       | DataSize    | 4            | Length of Data (N)                            |
+| 20       | Data        | N            | File data                                     |
+| 20+N     | Flags       | 1            | Bit flags                                     |
+| 21+N     | NameSize    | 1 (opt)      | Optional, if present                          |
+| 22+N     | Name        | M (opt)      | Optional, if present (M = NameSize)           |
+| ...      | MimeSize    | 1 (opt)      | Optional, if present                          |
+| ...      | Mime        | K (opt)      | Optional, if present (K = MimeSize)           |
+| ...      | LastModified| 5 (opt)      | Optional, if present                          |
+| ...      | Ttl         | 2 (opt)      | Optional, if present                          |
+| ...      | PairsSize   | 2 (opt)      | Optional, if present                          |
+| ...      | Pairs       | P (opt)      | Optional, if present (P = PairsSize)          |
+| ...      | Checksum    | 4            | CRC32                                         |
+| ...      | Padding     | 0-7          | To align to 8 bytes                           |
+
+### Version 3
+
+| Offset   | Field        | Size (bytes) | Description                                   |
+|----------|-------------|--------------|-----------------------------------------------|
+| 0        | Cookie      | 4            | Random number                                 |
+| 4        | Id          | 8            | Needle ID                                     |
+| 12       | Size        | 4            | Total size of the following fields            |
+| 16       | DataSize    | 4            | Length of Data (N)                            |
+| 20       | Data        | N            | File data                                     |
+| 20+N     | Flags       | 1            | Bit flags                                     |
+| 21+N     | NameSize    | 1 (opt)      | Optional, if present                          |
+| 22+N     | Name        | M (opt)      | Optional, if present (M = NameSize)           |
+| ...      | MimeSize    | 1 (opt)      | Optional, if present                          |
+| ...      | Mime        | K (opt)      | Optional, if present (K = MimeSize)           |
+| ...      | LastModified| 5 (opt)      | Optional, if present                          |
+| ...      | Ttl         | 2 (opt)      | Optional, if present                          |
+| ...      | PairsSize   | 2 (opt)      | Optional, if present                          |
+| ...      | Pairs       | P (opt)      | Optional, if present (P = PairsSize)          |
+| ...      | Checksum    | 4            | CRC32                                         |
+| ...      | Timestamp   | 8            | Append time in nanoseconds                    |
+| ...      | Padding     | 0-7          | To align to 8 bytes                           |
+
+- Offsets marked with `...` depend on the presence and size of previous optional fields.
+- Fields marked `(opt)` are optional and only present if the corresponding size or flag is non-zero.
+- N = DataSize, M = NameSize, K = MimeSize, P = PairsSize.
+
+---
+
+## Field Explanations
+
+- **Cookie**: 4 bytes, random value for security.
+- **Id**: 8 bytes, unique identifier for the Needle.
+- **Size**: 4 bytes, total size of the Needle data section (not including header, checksum, timestamp, or padding).
+- **DataSize**: 4 bytes, length of the Data field.
+- **Data**: File data (variable length).
+- **Flags**: 1 byte, bit flags for Needle properties.
+- **NameSize/Name**: 1 byte + variable, optional file name.
+- **MimeSize/Mime**: 1 byte + variable, optional MIME type.
+- **LastModified**: 5 bytes, optional last modified timestamp.
+- **Ttl**: 2 bytes, optional time-to-live.
+- **PairsSize/Pairs**: 2 bytes + variable, optional key-value pairs.
+- **Checksum**: 4 bytes, CRC32 checksum of the Needle data.
+- **Timestamp**: 8 bytes, append time (only in v3).
+- **Padding**: 0-7 bytes, to align the total Needle size to 8 bytes.
+
+---
+
+## Version Comparison Table
+
+| Field         | v1 | v2 | v3 |
+|---------------|----|----|----|
+| Cookie        | ✔  | ✔  | ✔  |
+| Id            | ✔  | ✔  | ✔  |
+| Size          | ✔  | ✔  | ✔  |
+| DataSize      |    | ✔  | ✔  |
+| Data          | ✔  | ✔  | ✔  |
+| Flags         |    | ✔  | ✔  |
+| NameSize/Name |    | ✔  | ✔  |
+| MimeSize/Mime |    | ✔  | ✔  |
+| LastModified  |    | ✔  | ✔  |
+| Ttl           |    | ✔  | ✔  |
+| PairsSize/Pairs|   | ✔  | ✔  |
+| Checksum      | ✔  | ✔  | ✔  |
+| Timestamp     |    |    | ✔  |
+| Padding       | ✔  | ✔  | ✔  |
+
+---
+
+## Flags Field Details
+
+The `Flags` field (present in v2 and v3) is a bitmask that encodes several boolean properties of the Needle. Each bit has a specific meaning:
+
+| Bit Value | Name                  | Meaning                                                      |
+|-----------|-----------------------|--------------------------------------------------------------|
+| 0x01      | FlagIsCompressed      | Data is compressed (isCompressed)                            |
+| 0x02      | FlagHasName           | Name field is present (NameSize/Name)                        |
+| 0x04      | FlagHasMime           | Mime field is present (MimeSize/Mime)                        |
+| 0x08      | FlagHasLastModifiedDate| LastModified field is present                                |
+| 0x10      | FlagHasTtl            | Ttl field is present                                         |
+| 0x20      | FlagHasPairs          | Pairs field is present (PairsSize/Pairs)                     |
+| 0x80      | FlagIsChunkManifest   | Data is a chunk manifest (for large files)                   |
+
+- If a flag is set, the corresponding field(s) will appear in the Needle layout at the appropriate position.
+- The `Flags` field is always present in v2 and v3, immediately after the Data field.
+
+## Optional Fields
+
+- Fields marked as optional in the layout tables are only present if the corresponding flag in the `Flags` field is set (except for Name/Mime/Pairs, which also depend on their size fields being non-zero).
+- The order of optional fields is fixed and matches the order of their flags.
+
+## Special Notes
+
+- **isCompressed**: If set, the Data field is compressed (typically using gzip). This is indicated by the lowest bit (0x01) in the Flags byte.
+- **isChunkManifest**: If set, the Data field contains a manifest describing chunks of a large file, not raw file data.
+- **All multi-byte fields** are stored in big-endian order.
+- **Padding** is always added at the end to align the total Needle size to 8 bytes.
+- **N = DataSize, M = NameSize, K = MimeSize, P = PairsSize** in the layout tables above.
+
+For more details, see the implementation in the corresponding Go files in this directory. 
@@ -3,12 +3,13 @@ package needle
 import (
 	"errors"
 	"fmt"
+	"io"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"io"
 )

 const (
@@ -52,7 +53,6 @@ func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, versi
 func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Version) (err error) {
 	n.ParseNeedleHeader(bytes)
 	if n.Size != size {
-		// cookie is not always passed in for this API. Use size to do preliminary checking.
 		if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) {
 			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc()
 			glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
@@ -61,30 +61,17 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio
 		stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc()
 		return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
 	}
-	switch version {
-	case Version1:
+	if version == Version1 {
 		n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
-	case Version2, Version3:
-		err = n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
-	}
-	if err != nil && err != io.EOF {
-		return err
-	}
-	if size > 0 {
-		checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
-		newChecksum := NewCRC(n.Data)
-		if checksum != newChecksum.Value() && checksum != uint32(newChecksum) {
-			// the crc.Value() function is to be deprecated. this double checking is for backward compatibility
-			// with seaweed version using crc.Value() instead of uint32(crc), which appears in commit 056c480eb
-			// and switch appeared in version 3.09.
-			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorCRC).Inc()
-			return errors.New("CRC error! Data On Disk Corrupted")
+	} else {
+		err := n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(size)])
+		if err != nil && err != io.EOF {
+			return err
 		}
-		n.Checksum = newChecksum
 	}
-	if version == Version3 {
-		tsOffset := NeedleHeaderSize + size + NeedleChecksumSize
-		n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize])
+	err = n.readNeedleTail(bytes[NeedleHeaderSize+size:], version)
+	if err != nil {
+		return err
 	}
 	return nil
 }
@@ -95,7 +82,6 @@ func (n *Needle) ReadData(r backend.BackendStorageFile, offset int64, size Size,
 	if err != nil {
 		return err
 	}
-
 	err = n.ReadBytes(bytes, offset, size, version)
 	if err == ErrorSizeMismatch && OffsetSize == 4 {
 		offset = offset + int64(MaxPossibleVolumeSize)
@@ -191,40 +177,24 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err

 func ReadNeedleHeader(r backend.BackendStorageFile, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) {
 	n = new(Needle)
-	if version == Version1 || version == Version2 || version == Version3 {
-		bytes = make([]byte, NeedleHeaderSize)

-		var count int
-		count, err = r.ReadAt(bytes, offset)
-		if err == io.EOF && count == NeedleHeaderSize {
-			err = nil
-		}
-		if count <= 0 || err != nil {
-			return nil, bytes, 0, err
-		}
+	bytes = make([]byte, NeedleHeaderSize)

-		n.ParseNeedleHeader(bytes)
-		bodyLength = NeedleBodyLength(n.Size, version)
+	var count int
+	count, err = r.ReadAt(bytes, offset)
+	if err == io.EOF && count == NeedleHeaderSize {
+		err = nil
 	}
+	if count <= 0 || err != nil {
+		return nil, bytes, 0, err
+	}
+
+	n.ParseNeedleHeader(bytes)
+	bodyLength = NeedleBodyLength(n.Size, version)

 	return
 }

-func PaddingLength(needleSize Size, version Version) Size {
-	if version == Version3 {
-		// this is same value as version2, but just listed here for clarity
-		return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize + TimestampSize) % NeedlePaddingSize)
-	}
-	return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize) % NeedlePaddingSize)
-}
-
-func NeedleBodyLength(needleSize Size, version Version) int64 {
-	if version == Version3 {
-		return int64(needleSize) + NeedleChecksumSize + TimestampSize + int64(PaddingLength(needleSize, version))
-	}
-	return int64(needleSize) + NeedleChecksumSize + int64(PaddingLength(needleSize, version))
-}
-
 // n should be a needle already read the header
 // the input stream will read until next file entry
 func (n *Needle) ReadNeedleBody(r backend.BackendStorageFile, version Version, offset int64, bodyLength int64) (bytes []byte, err error) {
@@ -255,14 +225,11 @@ func (n *Needle) ReadNeedleBodyBytes(needleBody []byte, version Version) (err er
 	switch version {
 	case Version1:
 		n.Data = needleBody[:n.Size]
-		n.Checksum = NewCRC(n.Data)
+		err = n.readNeedleTail(needleBody[n.Size:], version)
 	case Version2, Version3:
 		err = n.readNeedleDataVersion2(needleBody[0:n.Size])
-		n.Checksum = NewCRC(n.Data)
-
-		if version == Version3 {
-			tsOffset := n.Size + NeedleChecksumSize
-			n.AppendAtNs = util.BytesToUint64(needleBody[tsOffset : tsOffset+TimestampSize])
+		if err == nil {
+			err = n.readNeedleTail(needleBody[n.Size:], version)
 		}
 	default:
 		err = fmt.Errorf("unsupported version %d!", version)
@@ -2,11 +2,12 @@ package needle

 import (
 	"fmt"
+	"io"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"io"
 )

 // ReadNeedleData uses a needle without n.Data to read the content
@@ -78,10 +79,7 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
 		index, err = n.readNeedleDataVersion2NonData(metaSlice)
 	}

-	n.Checksum = CRC(util.BytesToUint32(metaSlice[index : index+NeedleChecksumSize]))
-	if version == Version3 {
-		n.AppendAtNs = util.BytesToUint64(metaSlice[index+NeedleChecksumSize : index+NeedleChecksumSize+TimestampSize])
-	}
+	err = n.readNeedleTail(metaSlice[index:], version)
 	return err

 }
@@ -0,0 +1,50 @@
+package needle
+
+import (
+	"errors"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (n *Needle) readNeedleTail(needleBody []byte, version Version) error {
+
+	// for all versions, we need to read the checksum
+	if len(n.Data) > 0 {
+		expectedChecksum := CRC(util.BytesToUint32(needleBody[0:NeedleChecksumSize]))
+		dataChecksum := NewCRC(n.Data)
+		if expectedChecksum != dataChecksum {
+			// the crc.Value() function is to be deprecated. this double checking is for backward compatibility
+			// with seaweed version using crc.Value() instead of uint32(crc), which appears in commit 056c480eb
+			// and switch appeared in version 3.09.
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorCRC).Inc()
+			return errors.New("CRC error! Data On Disk Corrupted")
+		}
+		n.Checksum = dataChecksum
+	} else {
+		// when data is skipped from reading, just read the checksum
+		n.Checksum = CRC(util.BytesToUint32(needleBody[0:NeedleChecksumSize]))
+	}
+
+	if version == Version3 {
+		tsOffset := NeedleChecksumSize
+		n.AppendAtNs = util.BytesToUint64(needleBody[tsOffset : tsOffset+TimestampSize])
+	}
+	return nil
+}
+
+func PaddingLength(needleSize Size, version Version) Size {
+	if version == Version3 {
+		// this is same value as version2, but just listed here for clarity
+		return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize + TimestampSize) % NeedlePaddingSize)
+	}
+	return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize) % NeedlePaddingSize)
+}
+
+func NeedleBodyLength(needleSize Size, version Version) int64 {
+	if version == Version3 {
+		return int64(needleSize) + NeedleChecksumSize + TimestampSize + int64(PaddingLength(needleSize, version))
+	}
+	return int64(needleSize) + NeedleChecksumSize + int64(PaddingLength(needleSize, version))
+}
@@ -3,7 +3,6 @@ package needle
 import (
 	"bytes"
 	"fmt"
-	"math"

 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
@@ -12,131 +11,35 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util/buffer_pool"
 )

-func (n *Needle) prepareWriteBuffer(version Version, writeBytes *bytes.Buffer) (Size, int64, error) {
-	writeBytes.Reset()
-	switch version {
-	case Version1:
-		header := make([]byte, NeedleHeaderSize)
-		CookieToBytes(header[0:CookieSize], n.Cookie)
-		NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
-		n.Size = Size(len(n.Data))
-		SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
-		size := n.Size
-		actualSize := NeedleHeaderSize + int64(n.Size)
-		writeBytes.Write(header)
-		writeBytes.Write(n.Data)
-		padding := PaddingLength(n.Size, version)
-		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
-		writeBytes.Write(header[0 : NeedleChecksumSize+padding])
-		return size, actualSize, nil
-	case Version2, Version3:
-		header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
-		CookieToBytes(header[0:CookieSize], n.Cookie)
-		NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
-		if len(n.Name) >= math.MaxUint8 {
-			n.NameSize = math.MaxUint8
-		} else {
-			n.NameSize = uint8(len(n.Name))
-		}
-		n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime))
-		if n.DataSize > 0 {
-			n.Size = 4 + Size(n.DataSize) + 1
-			if n.HasName() {
-				n.Size = n.Size + 1 + Size(n.NameSize)
-			}
-			if n.HasMime() {
-				n.Size = n.Size + 1 + Size(n.MimeSize)
-			}
-			if n.HasLastModifiedDate() {
-				n.Size = n.Size + LastModifiedBytesLength
-			}
-			if n.HasTtl() {
-				n.Size = n.Size + TtlBytesLength
-			}
-			if n.HasPairs() {
-				n.Size += 2 + Size(n.PairsSize)
-			}
-		} else {
-			n.Size = 0
-		}
-		SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
-		writeBytes.Write(header[0:NeedleHeaderSize])
-		if n.DataSize > 0 {
-			util.Uint32toBytes(header[0:4], n.DataSize)
-			writeBytes.Write(header[0:4])
-			writeBytes.Write(n.Data)
-			util.Uint8toBytes(header[0:1], n.Flags)
-			writeBytes.Write(header[0:1])
-			if n.HasName() {
-				util.Uint8toBytes(header[0:1], n.NameSize)
-				writeBytes.Write(header[0:1])
-				writeBytes.Write(n.Name[:n.NameSize])
-			}
-			if n.HasMime() {
-				util.Uint8toBytes(header[0:1], n.MimeSize)
-				writeBytes.Write(header[0:1])
-				writeBytes.Write(n.Mime)
-			}
-			if n.HasLastModifiedDate() {
-				util.Uint64toBytes(header[0:8], n.LastModified)
-				writeBytes.Write(header[8-LastModifiedBytesLength : 8])
-			}
-			if n.HasTtl() && n.Ttl != nil {
-				n.Ttl.ToBytes(header[0:TtlBytesLength])
-				writeBytes.Write(header[0:TtlBytesLength])
-			}
-			if n.HasPairs() {
-				util.Uint16toBytes(header[0:2], n.PairsSize)
-				writeBytes.Write(header[0:2])
-				writeBytes.Write(n.Pairs)
-			}
-		}
-		padding := PaddingLength(n.Size, version)
-		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
-		if version == Version2 {
-			writeBytes.Write(header[0 : NeedleChecksumSize+padding])
-		} else {
-			// version3
-			util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
-			writeBytes.Write(header[0 : NeedleChecksumSize+TimestampSize+padding])
-		}
-
-		return Size(n.DataSize), GetActualSize(n.Size, version), nil
-	}
-
-	return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
-}
-
 func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error) {
-
-	if end, _, e := w.GetStat(); e == nil {
-		defer func(w backend.BackendStorageFile, off int64) {
-			if err != nil {
-				if te := w.Truncate(end); te != nil {
-					glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
-				}
-			}
-		}(w, end)
-		offset = uint64(end)
-	} else {
+	end, _, e := w.GetStat()
+	if e != nil {
 		err = fmt.Errorf("Cannot Read Current Volume Position: %w", e)
 		return
 	}
+	offset = uint64(end)
 	if offset >= MaxPossibleVolumeSize && len(n.Data) != 0 {
 		err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize)
 		return
 	}
-
 	bytesBuffer := buffer_pool.SyncPoolGetBuffer()
-	defer buffer_pool.SyncPoolPutBuffer(bytesBuffer)
-
-	size, actualSize, err = n.prepareWriteBuffer(version, bytesBuffer)
-
-	if err == nil {
-		_, err = w.WriteAt(bytesBuffer.Bytes(), int64(offset))
+	defer func() {
 		if err != nil {
-			err = fmt.Errorf("failed to write %d bytes to %s at offset %d: %w", actualSize, w.Name(), offset, err)
+			if te := w.Truncate(end); te != nil {
+				// handle error or log
+			}
 		}
+		buffer_pool.SyncPoolPutBuffer(bytesBuffer)
+	}()
+
+	size, actualSize, err = writeNeedleByVersion(version, n, offset, bytesBuffer)
+	if err != nil {
+		return
+	}
+
+	_, err = w.WriteAt(bytesBuffer.Bytes(), int64(offset))
+	if err != nil {
+		err = fmt.Errorf("failed to write %d bytes to %s at offset %d: %w", actualSize, w.Name(), offset, err)
 	}

 	return offset, size, actualSize, err
@@ -170,3 +73,27 @@ func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size,
 	return

 }
+
+// prepareNeedleWrite encapsulates the common beginning logic for all versioned writeNeedle functions.
+func prepareNeedleWrite(w backend.BackendStorageFile, n *Needle) (offset uint64, bytesBuffer *bytes.Buffer, cleanup func(err error), err error) {
+	end, _, e := w.GetStat()
+	if e != nil {
+		err = fmt.Errorf("Cannot Read Current Volume Position: %w", e)
+		return
+	}
+	offset = uint64(end)
+	if offset >= MaxPossibleVolumeSize && len(n.Data) != 0 {
+		err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize)
+		return
+	}
+	bytesBuffer = buffer_pool.SyncPoolGetBuffer()
+	cleanup = func(err error) {
+		if err != nil {
+			if te := w.Truncate(end); te != nil {
+				// handle error or log
+			}
+		}
+		buffer_pool.SyncPoolPutBuffer(bytesBuffer)
+	}
+	return
+}
@@ -0,0 +1,142 @@
+package needle
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/buffer_pool"
+)
+
+func (n *Needle) LegacyPrepareWriteBuffer(version Version, writeBytes *bytes.Buffer) (Size, int64, error) {
+	writeBytes.Reset()
+	switch version {
+	case Version1:
+		header := make([]byte, NeedleHeaderSize)
+		CookieToBytes(header[0:CookieSize], n.Cookie)
+		NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+		n.Size = Size(len(n.Data))
+		SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+		size := n.Size
+		actualSize := NeedleHeaderSize + int64(n.Size)
+		writeBytes.Write(header)
+		writeBytes.Write(n.Data)
+		padding := PaddingLength(n.Size, version)
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		writeBytes.Write(header[0 : NeedleChecksumSize+padding])
+		return size, actualSize, nil
+	case Version2, Version3:
+		header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
+		CookieToBytes(header[0:CookieSize], n.Cookie)
+		NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+		if len(n.Name) >= math.MaxUint8 {
+			n.NameSize = math.MaxUint8
+		} else {
+			n.NameSize = uint8(len(n.Name))
+		}
+		n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime))
+		if n.DataSize > 0 {
+			n.Size = 4 + Size(n.DataSize) + 1
+			if n.HasName() {
+				n.Size = n.Size + 1 + Size(n.NameSize)
+			}
+			if n.HasMime() {
+				n.Size = n.Size + 1 + Size(n.MimeSize)
+			}
+			if n.HasLastModifiedDate() {
+				n.Size = n.Size + LastModifiedBytesLength
+			}
+			if n.HasTtl() {
+				n.Size = n.Size + TtlBytesLength
+			}
+			if n.HasPairs() {
+				n.Size += 2 + Size(n.PairsSize)
+			}
+		} else {
+			n.Size = 0
+		}
+		SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+		writeBytes.Write(header[0:NeedleHeaderSize])
+		if n.DataSize > 0 {
+			util.Uint32toBytes(header[0:4], n.DataSize)
+			writeBytes.Write(header[0:4])
+			writeBytes.Write(n.Data)
+			util.Uint8toBytes(header[0:1], n.Flags)
+			writeBytes.Write(header[0:1])
+			if n.HasName() {
+				util.Uint8toBytes(header[0:1], n.NameSize)
+				writeBytes.Write(header[0:1])
+				writeBytes.Write(n.Name[:n.NameSize])
+			}
+			if n.HasMime() {
+				util.Uint8toBytes(header[0:1], n.MimeSize)
+				writeBytes.Write(header[0:1])
+				writeBytes.Write(n.Mime)
+			}
+			if n.HasLastModifiedDate() {
+				util.Uint64toBytes(header[0:8], n.LastModified)
+				writeBytes.Write(header[8-LastModifiedBytesLength : 8])
+			}
+			if n.HasTtl() && n.Ttl != nil {
+				n.Ttl.ToBytes(header[0:TtlBytesLength])
+				writeBytes.Write(header[0:TtlBytesLength])
+			}
+			if n.HasPairs() {
+				util.Uint16toBytes(header[0:2], n.PairsSize)
+				writeBytes.Write(header[0:2])
+				writeBytes.Write(n.Pairs)
+			}
+		}
+		padding := PaddingLength(n.Size, version)
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		if version == Version2 {
+			writeBytes.Write(header[0 : NeedleChecksumSize+padding])
+		} else {
+			// version3
+			util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
+			writeBytes.Write(header[0 : NeedleChecksumSize+TimestampSize+padding])
+		}
+
+		return Size(n.DataSize), GetActualSize(n.Size, version), nil
+	}
+
+	return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
+}
+
+func (n *Needle) LegacyAppend(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error) {
+	if end, _, e := w.GetStat(); e == nil {
+		defer func(w backend.BackendStorageFile, off int64) {
+			if err != nil {
+				if te := w.Truncate(end); te != nil {
+					glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
+				}
+			}
+		}(w, end)
+		offset = uint64(end)
+	} else {
+		err = fmt.Errorf("Cannot Read Current Volume Position: %w", e)
+		return
+	}
+	if offset >= MaxPossibleVolumeSize && len(n.Data) != 0 {
+		err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize)
+		return
+	}
+
+	bytesBuffer := buffer_pool.SyncPoolGetBuffer()
+	defer buffer_pool.SyncPoolPutBuffer(bytesBuffer)
+
+	size, actualSize, err = n.LegacyPrepareWriteBuffer(version, bytesBuffer)
+
+	if err == nil {
+		_, err = w.WriteAt(bytesBuffer.Bytes(), int64(offset))
+		if err != nil {
+			err = fmt.Errorf("failed to write %d bytes to %s at offset %d: %w", actualSize, w.Name(), offset, err)
+		}
+	}
+
+	return offset, size, actualSize, err
+}
@@ -1,8 +1,10 @@
 package needle

 import (
+	"bytes"
 	"os"
 	"testing"
+	"time"

 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
@@ -62,3 +64,97 @@ func TestAppend(t *testing.T) {
 		t.Errorf("Fail to Append Needle.")
 	}
 }
+
+func versionString(v Version) string {
+	switch v {
+	case Version1:
+		return "Version1"
+	case Version2:
+		return "Version2"
+	case Version3:
+		return "Version3"
+	default:
+		return "UnknownVersion"
+	}
+}
+
+func TestWriteNeedle_CompatibilityWithLegacy(t *testing.T) {
+	versions := []Version{Version1, Version2, Version3}
+	for _, version := range versions {
+		t.Run(versionString(version), func(t *testing.T) {
+			n := &Needle{
+				Cookie:       0x12345678,
+				Id:           0x1122334455667788,
+				Data:         []byte("hello world"),
+				Flags:        0xFF,
+				Name:         []byte("filename.txt"),
+				Mime:         []byte("text/plain"),
+				LastModified: 0x1234567890,
+				Ttl:          nil, // Add TTL if needed
+				Pairs:        []byte("key=value"),
+				PairsSize:    9,
+				Checksum:     0xCAFEBABE,
+				AppendAtNs:   0xDEADBEEF,
+			}
+
+			// Legacy
+			legacyBuf := &bytes.Buffer{}
+			_, _, err := n.LegacyPrepareWriteBuffer(version, legacyBuf)
+			if err != nil {
+				t.Fatalf("LegacyPrepareWriteBuffer failed: %v", err)
+			}
+
+			// New
+			newBuf := &bytes.Buffer{}
+			offset := uint64(0)
+			switch version {
+			case Version1:
+				_, _, err = writeNeedleV1(n, offset, newBuf)
+			case Version2:
+				_, _, err = writeNeedleV2(n, offset, newBuf)
+			case Version3:
+				_, _, err = writeNeedleV3(n, offset, newBuf)
+			}
+			if err != nil {
+				t.Fatalf("writeNeedleV%d failed: %v", version, err)
+			}
+
+			if !bytes.Equal(legacyBuf.Bytes(), newBuf.Bytes()) {
+				t.Errorf("Data layout mismatch for version %d\nLegacy: %x\nNew:    %x", version, legacyBuf.Bytes(), newBuf.Bytes())
+			}
+		})
+	}
+}
+
+type mockBackendWriter struct {
+	buf *bytes.Buffer
+}
+
+func (m *mockBackendWriter) WriteAt(p []byte, off int64) (n int, err error) {
+	return m.buf.Write(p)
+}
+
+func (m *mockBackendWriter) GetStat() (int64, time.Time, error) {
+	return 0, time.Time{}, nil
+}
+
+func (m *mockBackendWriter) Truncate(size int64) error {
+	return nil
+}
+
+func (m *mockBackendWriter) Name() string {
+	return "mock"
+}
+
+func (m *mockBackendWriter) Close() error {
+	return nil
+}
+
+func (m *mockBackendWriter) Sync() error {
+	return nil
+}
+
+func (m *mockBackendWriter) ReadAt(p []byte, off int64) (n int, err error) {
+	// Not used in this test
+	return 0, nil
+}
@@ -0,0 +1,26 @@
+package needle
+
+import (
+	"bytes"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func writeNeedleV1(n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	bytesBuffer.Reset()
+	header := make([]byte, NeedleHeaderSize)
+	CookieToBytes(header[0:CookieSize], n.Cookie)
+	NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+	n.Size = Size(len(n.Data))
+	SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+	size = n.Size
+	actualSize = NeedleHeaderSize + int64(n.Size)
+	bytesBuffer.Write(header)
+	bytesBuffer.Write(n.Data)
+	padding := PaddingLength(n.Size, Version1)
+	util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+	bytesBuffer.Write(header[0 : NeedleChecksumSize+padding])
+
+	return size, actualSize, nil
+}
@@ -0,0 +1,86 @@
+package needle
+
+import (
+	"bytes"
+	"math"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func writeNeedleV2(n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	return writeNeedleCommon(n, offset, bytesBuffer, Version2, func(n *Needle, header []byte, bytesBuffer *bytes.Buffer, padding int) {
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		bytesBuffer.Write(header[0 : NeedleChecksumSize+padding])
+	})
+}
+
+func writeNeedleCommon(n *Needle, offset uint64, bytesBuffer *bytes.Buffer, version Version, writeFooter func(n *Needle, header []byte, bytesBuffer *bytes.Buffer, padding int)) (size Size, actualSize int64, err error) {
+	bytesBuffer.Reset()
+	header := make([]byte, NeedleHeaderSize+TimestampSize)
+	CookieToBytes(header[0:CookieSize], n.Cookie)
+	NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+	if len(n.Name) >= math.MaxUint8 {
+		n.NameSize = math.MaxUint8
+	} else {
+		n.NameSize = uint8(len(n.Name))
+	}
+	n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime))
+	if n.DataSize > 0 {
+		n.Size = 4 + Size(n.DataSize) + 1
+		if n.HasName() {
+			n.Size = n.Size + 1 + Size(n.NameSize)
+		}
+		if n.HasMime() {
+			n.Size = n.Size + 1 + Size(n.MimeSize)
+		}
+		if n.HasLastModifiedDate() {
+			n.Size = n.Size + LastModifiedBytesLength
+		}
+		if n.HasTtl() {
+			n.Size = n.Size + TtlBytesLength
+		}
+		if n.HasPairs() {
+			n.Size += 2 + Size(n.PairsSize)
+		}
+	} else {
+		n.Size = 0
+	}
+	SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+	bytesBuffer.Write(header[0:NeedleHeaderSize])
+	if n.DataSize > 0 {
+		util.Uint32toBytes(header[0:4], n.DataSize)
+		bytesBuffer.Write(header[0:4])
+		bytesBuffer.Write(n.Data)
+		util.Uint8toBytes(header[0:1], n.Flags)
+		bytesBuffer.Write(header[0:1])
+		if n.HasName() {
+			util.Uint8toBytes(header[0:1], n.NameSize)
+			bytesBuffer.Write(header[0:1])
+			bytesBuffer.Write(n.Name[:n.NameSize])
+		}
+		if n.HasMime() {
+			util.Uint8toBytes(header[0:1], n.MimeSize)
+			bytesBuffer.Write(header[0:1])
+			bytesBuffer.Write(n.Mime)
+		}
+		if n.HasLastModifiedDate() {
+			util.Uint64toBytes(header[0:8], n.LastModified)
+			bytesBuffer.Write(header[8-LastModifiedBytesLength : 8])
+		}
+		if n.HasTtl() && n.Ttl != nil {
+			n.Ttl.ToBytes(header[0:TtlBytesLength])
+			bytesBuffer.Write(header[0:TtlBytesLength])
+		}
+		if n.HasPairs() {
+			util.Uint16toBytes(header[0:2], n.PairsSize)
+			bytesBuffer.Write(header[0:2])
+			bytesBuffer.Write(n.Pairs)
+		}
+	}
+	padding := PaddingLength(n.Size, version)
+	writeFooter(n, header, bytesBuffer, int(padding))
+	size = Size(n.DataSize)
+	actualSize = GetActualSize(n.Size, version)
+	return size, actualSize, nil
+}
@@ -0,0 +1,16 @@
+package needle
+
+import (
+	"bytes"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func writeNeedleV3(n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	return writeNeedleCommon(n, offset, bytesBuffer, Version3, func(n *Needle, header []byte, bytesBuffer *bytes.Buffer, padding int) {
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
+		bytesBuffer.Write(header[0 : NeedleChecksumSize+TimestampSize+padding])
+	})
+}
@@ -0,0 +1,23 @@
+package needle
+
+import (
+	"bytes"
+	"fmt"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func writeNeedleByVersion(version Version, n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	// Switch logic moved from needle_write.go
+	switch version {
+	case Version1:
+		size, actualSize, err = writeNeedleV1(n, offset, bytesBuffer)
+	case Version2:
+		size, actualSize, err = writeNeedleV2(n, offset, bytesBuffer)
+	case Version3:
+		size, actualSize, err = writeNeedleV3(n, offset, bytesBuffer)
+	default:
+		err = fmt.Errorf("unsupported version: %d", version)
+	}
+	return
+}
@@ -1,330 +1,284 @@
 package needle_map

+/* CompactMap is an in-memory map of needle indeces, optimized for memory usage.
+ *
+ * It's implemented as a map of sorted indeces segments, which are in turn accessed through binary
+ * search. This guarantees a best-case scenario (ordered inserts/updates) of O(1) and a worst case
+ * scenario of O(log n) runtime, with memory usage unaffected by insert ordering.
+ *
+ * Note that even at O(log n), the clock time for both reads and writes is very low, so CompactMap
+ * will seldom bottleneck index operations.
+ */
+
 import (
+	"fmt"
+	"math"
+	"slices"
 	"sort"
 	"sync"

-	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )

 const (
-	MaxSectionBucketSize = 1024 * 8
-	LookBackWindowSize   = 1024 // how many entries to look back when inserting into a section
+	MaxCompactKey    = math.MaxUint16
+	SegmentChunkSize = 50000 // should be <= MaxCompactKey
 )

-type SectionalNeedleId uint32
-
-const SectionalNeedleIdLimit = 1<<32 - 1
-
-type SectionalNeedleValue struct {
-	Key          SectionalNeedleId
-	OffsetLower  OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
-	Size         Size        `comment:"Size of the data portion"`
-	OffsetHigher OffsetHigher
+type CompactKey uint16
+type CompactOffset [types.OffsetSize]byte
+type CompactNeedleValue struct {
+	key    CompactKey
+	offset CompactOffset
+	size   types.Size
 }

-type CompactSection struct {
+type Chunk uint64
+type CompactMapSegment struct {
+	list     []CompactNeedleValue
+	chunk    Chunk
+	firstKey CompactKey
+	lastKey  CompactKey
+}
+
+type CompactMap struct {
 	sync.RWMutex
-	values   []SectionalNeedleValue
-	overflow Overflow
-	start    NeedleId
-	end      NeedleId
+
+	segments map[Chunk]*CompactMapSegment
 }

-type Overflow []SectionalNeedleValue
+func (ck CompactKey) Key(chunk Chunk) types.NeedleId {
+	return (types.NeedleId(SegmentChunkSize) * types.NeedleId(chunk)) + types.NeedleId(ck)
+}

-func NewCompactSection(start NeedleId) *CompactSection {
-	return &CompactSection{
-		values:   make([]SectionalNeedleValue, 0),
-		overflow: Overflow(make([]SectionalNeedleValue, 0)),
-		start:    start,
+func OffsetToCompact(offset types.Offset) CompactOffset {
+	var co CompactOffset
+	types.OffsetToBytes(co[:], offset)
+	return co
+}
+
+func (co CompactOffset) Offset() types.Offset {
+	return types.BytesToOffset(co[:])
+}
+
+func (cnv CompactNeedleValue) NeedleValue(chunk Chunk) NeedleValue {
+	return NeedleValue{
+		Key:    cnv.key.Key(chunk),
+		Offset: cnv.offset.Offset(),
+		Size:   cnv.size,
 	}
 }

-// return old entry size
-func (cs *CompactSection) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) {
-	cs.Lock()
-	defer cs.Unlock()
-
-	if key > cs.end {
-		cs.end = key
+func newCompactMapSegment(chunk Chunk) *CompactMapSegment {
+	return &CompactMapSegment{
+		list:     []CompactNeedleValue{},
+		chunk:    chunk,
+		firstKey: MaxCompactKey,
+		lastKey:  0,
 	}
-	skey := SectionalNeedleId(key - cs.start)
-	if i := cs.binarySearchValues(skey); i >= 0 {
+}
+
+func (cs *CompactMapSegment) len() int {
+	return len(cs.list)
+}
+
+func (cs *CompactMapSegment) cap() int {
+	return cap(cs.list)
+}
+
+func (cs *CompactMapSegment) compactKey(key types.NeedleId) CompactKey {
+	return CompactKey(key - (types.NeedleId(SegmentChunkSize) * types.NeedleId(cs.chunk)))
+}
+
+// bsearchKey returns the CompactNeedleValue index for a given ID key.
+// If the key is not found, it returns the index where it should be inserted instead.
+func (cs *CompactMapSegment) bsearchKey(key types.NeedleId) (int, bool) {
+	ck := cs.compactKey(key)
+
+	switch {
+	case len(cs.list) == 0:
+		return 0, false
+	case ck == cs.firstKey:
+		return 0, true
+	case ck <= cs.firstKey:
+		return 0, false
+	case ck == cs.lastKey:
+		return len(cs.list) - 1, true
+	case ck > cs.lastKey:
+		return len(cs.list), false
+	}
+
+	i := sort.Search(len(cs.list), func(i int) bool {
+		return cs.list[i].key >= ck
+	})
+	return i, cs.list[i].key == ck
+}
+
+// set inserts/updates a CompactNeedleValue.
+// If the operation is an update, returns the overwritten value's previous offset and size.
+func (cs *CompactMapSegment) set(key types.NeedleId, offset types.Offset, size types.Size) (oldOffset types.Offset, oldSize types.Size) {
+	i, found := cs.bsearchKey(key)
+	if found {
 		// update
-		oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.values[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size
-		cs.values[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size
+		o := cs.list[i].offset.Offset()
+		oldOffset.OffsetLower = o.OffsetLower
+		oldOffset.OffsetHigher = o.OffsetHigher
+		oldSize = cs.list[i].size
+
+		o.OffsetLower = offset.OffsetLower
+		o.OffsetHigher = offset.OffsetHigher
+		cs.list[i].offset = OffsetToCompact(o)
+		cs.list[i].size = size
 		return
 	}

-	var lkey SectionalNeedleId
-	if len(cs.values) > 0 {
-		lkey = cs.values[len(cs.values)-1].Key
+	// insert
+	if len(cs.list) >= SegmentChunkSize {
+		panic(fmt.Sprintf("attempted to write more than %d entries on CompactMapSegment %p!!!", SegmentChunkSize, cs))
 	}
-
-	hasAdded := false
-	switch {
-	case len(cs.values) < MaxSectionBucketSize && lkey <= skey:
-		// non-overflow insert
-		cs.values = append(cs.values, SectionalNeedleValue{
-			Key:          skey,
-			OffsetLower:  offset.OffsetLower,
-			Size:         size,
-			OffsetHigher: offset.OffsetHigher,
-		})
-		hasAdded = true
-	case len(cs.values) < MaxSectionBucketSize:
-		// still has capacity and only partially out of order
-		lookBackIndex := len(cs.values) - LookBackWindowSize
-		if lookBackIndex < 0 {
-			lookBackIndex = 0
-		}
-		if cs.values[lookBackIndex].Key <= skey {
-			for ; lookBackIndex < len(cs.values); lookBackIndex++ {
-				if cs.values[lookBackIndex].Key >= skey {
-					break
-				}
-			}
-			cs.values = append(cs.values, SectionalNeedleValue{})
-			copy(cs.values[lookBackIndex+1:], cs.values[lookBackIndex:])
-			cs.values[lookBackIndex].Key, cs.values[lookBackIndex].Size = skey, size
-			cs.values[lookBackIndex].OffsetLower, cs.values[lookBackIndex].OffsetHigher = offset.OffsetLower, offset.OffsetHigher
-			hasAdded = true
-		}
-	}
-
-	// overflow insert
-	if !hasAdded {
-		if oldValue, found := cs.findOverflowEntry(skey); found {
-			oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValue.OffsetHigher, oldValue.OffsetLower, oldValue.Size
-		}
-		cs.setOverflowEntry(skey, offset, size)
+	if len(cs.list) == SegmentChunkSize-1 {
+		// if we max out our segment storage, pin its capacity to minimize memory usage
+		nl := make([]CompactNeedleValue, SegmentChunkSize, SegmentChunkSize)
+		copy(nl, cs.list[:i])
+		copy(nl[i+1:], cs.list[i:])
+		cs.list = nl
 	} else {
-		// if we maxed out our values bucket, pin its capacity to minimize memory usage
-		if len(cs.values) == MaxSectionBucketSize {
-			bucket := make([]SectionalNeedleValue, len(cs.values))
-			copy(bucket, cs.values)
-			cs.values = bucket
-		}
+		cs.list = append(cs.list, CompactNeedleValue{})
+		copy(cs.list[i+1:], cs.list[i:])
+	}
+
+	ck := cs.compactKey(key)
+	cs.list[i] = CompactNeedleValue{
+		key:    ck,
+		offset: OffsetToCompact(offset),
+		size:   size,
+	}
+	if ck < cs.firstKey {
+		cs.firstKey = ck
+	}
+	if ck > cs.lastKey {
+		cs.lastKey = ck
 	}

 	return
 }

-func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size Size) {
-	needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size, OffsetHigher: offset.OffsetHigher}
-	insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
-		return cs.overflow[i].Key >= needleValue.Key
-	})
-
-	if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key {
-		cs.overflow[insertCandidate] = needleValue
-		return
+// get seeks a map entry by key. Returns an entry pointer, with a boolean specifiying if the entry was found.
+func (cs *CompactMapSegment) get(key types.NeedleId) (*CompactNeedleValue, bool) {
+	if i, found := cs.bsearchKey(key); found {
+		return &cs.list[i], true
 	}

-	cs.overflow = append(cs.overflow, SectionalNeedleValue{})
-	copy(cs.overflow[insertCandidate+1:], cs.overflow[insertCandidate:])
-	cs.overflow[insertCandidate] = needleValue
+	return nil, false
 }

-func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nv SectionalNeedleValue, found bool) {
-	foundCandidate := sort.Search(len(cs.overflow), func(i int) bool {
-		return cs.overflow[i].Key >= key
-	})
-	if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key {
-		return cs.overflow[foundCandidate], true
-	}
-	return nv, false
-}
-
-func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
-	length := len(cs.overflow)
-	deleteCandidate := sort.Search(length, func(i int) bool {
-		return cs.overflow[i].Key >= key
-	})
-	if deleteCandidate != length && cs.overflow[deleteCandidate].Key == key {
-		if cs.overflow[deleteCandidate].Size.IsValid() {
-			cs.overflow[deleteCandidate].Size = -cs.overflow[deleteCandidate].Size
+// delete deletes a map entry by key. Returns the entries' previous Size, if available.
+func (cs *CompactMapSegment) delete(key types.NeedleId) types.Size {
+	if i, found := cs.bsearchKey(key); found {
+		if cs.list[i].size > 0 && cs.list[i].size.IsValid() {
+			ret := cs.list[i].size
+			cs.list[i].size = -cs.list[i].size
+			return ret
 		}
 	}
+
+	return types.Size(0)
 }

-// return old entry size
-func (cs *CompactSection) Delete(key NeedleId) Size {
-	cs.Lock()
-	defer cs.Unlock()
-	ret := Size(0)
-	if key > cs.end {
-		return ret
+func NewCompactMap() *CompactMap {
+	return &CompactMap{
+		segments: map[Chunk]*CompactMapSegment{},
 	}
-	skey := SectionalNeedleId(key - cs.start)
-	if i := cs.binarySearchValues(skey); i >= 0 {
-		if cs.values[i].Size > 0 && cs.values[i].Size.IsValid() {
-			ret = cs.values[i].Size
-			cs.values[i].Size = -cs.values[i].Size
-		}
-	}
-	if v, found := cs.findOverflowEntry(skey); found {
-		cs.deleteOverflowEntry(skey)
-		ret = v.Size
-	}
-	return ret
 }
-func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
-	cs.RLock()
-	defer cs.RUnlock()
-	if key > cs.end {
-		return nil, false
+
+func (cm *CompactMap) Len() int {
+	l := 0
+	for _, s := range cm.segments {
+		l += s.len()
 	}
-	skey := SectionalNeedleId(key - cs.start)
-	if v, ok := cs.findOverflowEntry(skey); ok {
-		nv := toNeedleValue(v, cs)
-		return &nv, true
+	return l
+}
+
+func (cm *CompactMap) Cap() int {
+	c := 0
+	for _, s := range cm.segments {
+		c += s.cap()
 	}
-	if i := cs.binarySearchValues(skey); i >= 0 {
-		nv := toNeedleValue(cs.values[i], cs)
+	return c
+}
+
+func (cm *CompactMap) String() string {
+	if cm.Len() == 0 {
+		return "empty"
+	}
+	return fmt.Sprintf(
+		"%d/%d elements on %d segments, %.02f%% efficiency",
+		cm.Len(), cm.Cap(), len(cm.segments),
+		float64(100)*float64(cm.Len())/float64(cm.Cap()))
+}
+
+func (cm *CompactMap) segmentForKey(key types.NeedleId) *CompactMapSegment {
+	chunk := Chunk(key / SegmentChunkSize)
+	if cs, ok := cm.segments[chunk]; ok {
+		return cs
+	}
+
+	cs := newCompactMapSegment(chunk)
+	cm.segments[chunk] = cs
+	return cs
+}
+
+// Set inserts/updates a NeedleValue.
+// If the operation is an update, returns the overwritten value's previous offset and size.
+func (cm *CompactMap) Set(key types.NeedleId, offset types.Offset, size types.Size) (oldOffset types.Offset, oldSize types.Size) {
+	cm.RLock()
+	defer cm.RUnlock()
+
+	cs := cm.segmentForKey(key)
+	return cs.set(key, offset, size)
+}
+
+// Get seeks a map entry by key. Returns an entry pointer, with a boolean specifiying if the entry was found.
+func (cm *CompactMap) Get(key types.NeedleId) (*NeedleValue, bool) {
+	cm.RLock()
+	defer cm.RUnlock()
+
+	cs := cm.segmentForKey(key)
+	if cnv, found := cs.get(key); found {
+		nv := cnv.NeedleValue(cs.chunk)
 		return &nv, true
 	}
 	return nil, false
 }
-func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {
-	x := sort.Search(len(cs.values), func(i int) bool {
-		return cs.values[i].Key >= key
-	})
-	if x >= len(cs.values) {
-		return -1
-	}
-	if cs.values[x].Key > key {
-		return -2
-	}
-	return x
+
+// Delete deletes a map entry by key. Returns the entries' previous Size, if available.
+func (cm *CompactMap) Delete(key types.NeedleId) types.Size {
+	cm.RLock()
+	defer cm.RUnlock()
+
+	cs := cm.segmentForKey(key)
+	return cs.delete(key)
 }

-// This map assumes mostly inserting increasing keys
-// This map assumes mostly inserting increasing keys
-type CompactMap struct {
-	list []*CompactSection
-}
-
-func NewCompactMap() *CompactMap {
-	return &CompactMap{}
-}
-
-func (cm *CompactMap) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) {
-	x := cm.binarySearchCompactSection(key)
-	if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
-		// println(x, "adding to existing", len(cm.list), "sections, starting", key)
-		cs := NewCompactSection(key)
-		cm.list = append(cm.list, cs)
-		x = len(cm.list) - 1
-		//keep compact section sorted by start
-		for x >= 0 {
-			if x > 0 && cm.list[x-1].start > key {
-				cm.list[x] = cm.list[x-1]
-				// println("shift", x, "start", cs.start, "to", x-1)
-				x = x - 1
-			} else {
-				cm.list[x] = cs
-				// println("cs", x, "start", cs.start)
-				break
-			}
-		}
-	}
-	// println(key, "set to section[", x, "].start", cm.list[x].start)
-	return cm.list[x].Set(key, offset, size)
-}
-func (cm *CompactMap) Delete(key NeedleId) Size {
-	x := cm.binarySearchCompactSection(key)
-	if x < 0 {
-		return Size(0)
-	}
-	return cm.list[x].Delete(key)
-}
-func (cm *CompactMap) Get(key NeedleId) (*NeedleValue, bool) {
-	x := cm.binarySearchCompactSection(key)
-	if x < 0 {
-		return nil, false
-	}
-	return cm.list[x].Get(key)
-}
-func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
-	l, h := 0, len(cm.list)-1
-	if h < 0 {
-		return -5
-	}
-	if cm.list[h].start <= key {
-		if len(cm.list[h].values) < MaxSectionBucketSize || key <= cm.list[h].end {
-			return h
-		}
-		return -4
-	}
-	for l <= h {
-		m := (l + h) / 2
-		if key < cm.list[m].start {
-			h = m - 1
-		} else { // cm.list[m].start <= key
-			if cm.list[m+1].start <= key {
-				l = m + 1
-			} else {
-				return m
-			}
-		}
-	}
-	return -3
-}
-
-// Visit visits all entries or stop if any error when visiting
+// AscendingVisit runs a function on all entries, in ascending key order. Returns any errors hit while visiting.
 func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
-	for _, cs := range cm.list {
-		cs.RLock()
-		var i, j int
-		for i, j = 0, 0; i < len(cs.overflow) && j < len(cs.values); {
-			if cs.overflow[i].Key < cs.values[j].Key {
-				if err := visit(toNeedleValue(cs.overflow[i], cs)); err != nil {
-					cs.RUnlock()
-					return err
-				}
-				i++
-			} else if cs.overflow[i].Key == cs.values[j].Key {
-				j++
-			} else {
-				if err := visit(toNeedleValue(cs.values[j], cs)); err != nil {
-					cs.RUnlock()
-					return err
-				}
-				j++
-			}
-		}
-		for ; i < len(cs.overflow); i++ {
-			if err := visit(toNeedleValue(cs.overflow[i], cs)); err != nil {
-				cs.RUnlock()
+	cm.RLock()
+	defer cm.RUnlock()
+
+	chunks := []Chunk{}
+	for c := range cm.segments {
+		chunks = append(chunks, c)
+	}
+	slices.Sort(chunks)
+
+	for _, c := range chunks {
+		cs := cm.segments[c]
+		for _, cnv := range cs.list {
+			nv := cnv.NeedleValue(cs.chunk)
+			if err := visit(nv); err != nil {
 				return err
 			}
 		}
-		for ; j < len(cs.values); j++ {
-			if err := visit(toNeedleValue(cs.values[j], cs)); err != nil {
-				cs.RUnlock()
-				return err
-			}
-		}
-		cs.RUnlock()
 	}
 	return nil
 }
-
-func toNeedleValue(snv SectionalNeedleValue, cs *CompactSection) NeedleValue {
-	offset := Offset{
-		OffsetHigher: snv.OffsetHigher,
-		OffsetLower:  snv.OffsetLower,
-	}
-	return NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size}
-}
-
-func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) SectionalNeedleValue {
-	return SectionalNeedleValue{
-		Key:          SectionalNeedleId(nv.Key - cs.start),
-		OffsetLower:  nv.Offset.OffsetLower,
-		Size:         nv.Size,
-		OffsetHigher: nv.Offset.OffsetHigher,
-	}
-}
@@ -5,11 +5,12 @@ package needle_map

 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/storage/types"
-	"github.com/stretchr/testify/assert"
 	"log"
 	"os"
 	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/stretchr/testify/assert"
 )

 func Test5bytesIndexLoading(t *testing.T) {
@@ -43,6 +43,7 @@ func TestMemoryUsage(t *testing.T) {

 		PrintMemUsage(totalRowCount)
 		now := time.Now()
+		fmt.Printf("\tCompactMap = %s", m.String())
 		fmt.Printf("\tTaken = %v\n", now.Sub(startTime))
 		startTime = now
 	}
@@ -2,240 +2,476 @@ package needle_map

 import (
 	"fmt"
-	"log"
-	"os"
+	"math/rand"
+	"reflect"
 	"testing"

-	"github.com/seaweedfs/seaweedfs/weed/sequence"
-	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )

-func TestSnowflakeSequencer(t *testing.T) {
-	m := NewCompactMap()
-	seq, _ := sequence.NewSnowflakeSequencer("for_test", 1)
+func TestSegmentBsearchKey(t *testing.T) {
+	testSegment := &CompactMapSegment{
+		list: []CompactNeedleValue{
+			CompactNeedleValue{key: 10},
+			CompactNeedleValue{key: 20},
+			CompactNeedleValue{key: 21},
+			CompactNeedleValue{key: 26},
+			CompactNeedleValue{key: 30},
+		},
+		firstKey: 10,
+		lastKey:  30,
+	}

-	for i := 0; i < 200000; i++ {
-		id := seq.NextFileId(1)
-		oldOffset, oldSize := m.Set(NeedleId(id), ToOffset(8), 3000073)
-		if oldSize != 0 {
-			t.Errorf("id %d oldOffset %v oldSize %d", id, oldOffset, oldSize)
+	testCases := []struct {
+		name      string
+		cs        *CompactMapSegment
+		key       types.NeedleId
+		wantIndex int
+		wantFound bool
+	}{
+		{
+			name:      "empty segment",
+			cs:        newCompactMapSegment(0),
+			key:       123,
+			wantIndex: 0,
+			wantFound: false,
+		},
+		{
+			name:      "new key, insert at beggining",
+			cs:        testSegment,
+			key:       5,
+			wantIndex: 0,
+			wantFound: false,
+		},
+		{
+			name:      "new key, insert at end",
+			cs:        testSegment,
+			key:       100,
+			wantIndex: 5,
+			wantFound: false,
+		},
+		{
+			name:      "new key, insert second",
+			cs:        testSegment,
+			key:       12,
+			wantIndex: 1,
+			wantFound: false,
+		},
+		{
+			name:      "new key, insert in middle",
+			cs:        testSegment,
+			key:       23,
+			wantIndex: 3,
+			wantFound: false,
+		},
+		{
+			name:      "key #1",
+			cs:        testSegment,
+			key:       10,
+			wantIndex: 0,
+			wantFound: true,
+		},
+		{
+			name:      "key #2",
+			cs:        testSegment,
+			key:       20,
+			wantIndex: 1,
+			wantFound: true,
+		},
+		{
+			name:      "key #3",
+			cs:        testSegment,
+			key:       21,
+			wantIndex: 2,
+			wantFound: true,
+		},
+		{
+			name:      "key #4",
+			cs:        testSegment,
+			key:       26,
+			wantIndex: 3,
+			wantFound: true,
+		},
+		{
+			name:      "key #5",
+			cs:        testSegment,
+			key:       30,
+			wantIndex: 4,
+			wantFound: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			index, found := tc.cs.bsearchKey(tc.key)
+			if got, want := index, tc.wantIndex; got != want {
+				t.Errorf("expected %v, got %v", want, got)
+			}
+			if got, want := found, tc.wantFound; got != want {
+				t.Errorf("expected %v, got %v", want, got)
+			}
+		})
+	}
+}
+
+func TestSegmentSet(t *testing.T) {
+	testSegment := &CompactMapSegment{
+		list: []CompactNeedleValue{
+			CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
+			CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
+			CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
+		},
+		firstKey: 10,
+		lastKey:  30,
+	}
+
+	if got, want := testSegment.len(), 3; got != want {
+		t.Errorf("got starting size %d, want %d", got, want)
+	}
+	if got, want := testSegment.cap(), 3; got != want {
+		t.Errorf("got starting capacity %d, want %d", got, want)
+	}
+
+	testSets := []struct {
+		name       string
+		key        types.NeedleId
+		offset     types.Offset
+		size       types.Size
+		wantOffset types.Offset
+		wantSize   types.Size
+	}{
+		{
+			name: "insert at beggining",
+			key:  5, offset: types.Uint32ToOffset(1000), size: 123,
+			wantOffset: types.Uint32ToOffset(0), wantSize: 0,
+		},
+		{
+			name: "insert at end",
+			key:  51, offset: types.Uint32ToOffset(7000), size: 456,
+			wantOffset: types.Uint32ToOffset(0), wantSize: 0,
+		},
+		{
+			name: "insert in middle",
+			key:  25, offset: types.Uint32ToOffset(8000), size: 789,
+			wantOffset: types.Uint32ToOffset(0), wantSize: 0,
+		},
+		{
+			name: "update existing",
+			key:  30, offset: types.Uint32ToOffset(9000), size: 999,
+			wantOffset: types.Uint32ToOffset(300), wantSize: 300,
+		},
+	}
+
+	for _, ts := range testSets {
+		offset, size := testSegment.set(ts.key, ts.offset, ts.size)
+		if offset != ts.wantOffset {
+			t.Errorf("%s: got offset %v, want %v", ts.name, offset, ts.wantOffset)
+		}
+		if size != ts.wantSize {
+			t.Errorf("%s: got size %v, want %v", ts.name, size, ts.wantSize)
 		}
 	}

+	wantSegment := &CompactMapSegment{
+		list: []CompactNeedleValue{
+			CompactNeedleValue{key: 5, offset: OffsetToCompact(types.Uint32ToOffset(1000)), size: 123},
+			CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
+			CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
+			CompactNeedleValue{key: 25, offset: OffsetToCompact(types.Uint32ToOffset(8000)), size: 789},
+			CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(9000)), size: 999},
+			CompactNeedleValue{key: 51, offset: OffsetToCompact(types.Uint32ToOffset(7000)), size: 456},
+		},
+		firstKey: 5,
+		lastKey:  51,
+	}
+	if !reflect.DeepEqual(testSegment, wantSegment) {
+		t.Errorf("got result segment %v, want %v", testSegment, wantSegment)
+	}
+
+	if got, want := testSegment.len(), 6; got != want {
+		t.Errorf("got result size %d, want %d", got, want)
+	}
+	if got, want := testSegment.cap(), 6; got != want {
+		t.Errorf("got result capacity %d, want %d", got, want)
+	}
 }

-func TestOverflow2(t *testing.T) {
-	m := NewCompactMap()
-	_, oldSize := m.Set(NeedleId(150088), ToOffset(8), 3000073)
-	if oldSize != 0 {
-		t.Fatalf("expecting no previous data")
+func TestSegmentSetOrdering(t *testing.T) {
+	keys := []types.NeedleId{}
+	for i := 0; i < SegmentChunkSize; i++ {
+		keys = append(keys, types.NeedleId(i))
 	}
-	_, oldSize = m.Set(NeedleId(150088), ToOffset(8), 3000073)
-	if oldSize != 3000073 {
-		t.Fatalf("expecting previous data size is %d, not %d", 3000073, oldSize)
-	}
-	m.Set(NeedleId(150073), ToOffset(8), 3000073)
-	m.Set(NeedleId(150089), ToOffset(8), 3000073)
-	m.Set(NeedleId(150076), ToOffset(8), 3000073)
-	m.Set(NeedleId(150124), ToOffset(8), 3000073)
-	m.Set(NeedleId(150137), ToOffset(8), 3000073)
-	m.Set(NeedleId(150147), ToOffset(8), 3000073)
-	m.Set(NeedleId(150145), ToOffset(8), 3000073)
-	m.Set(NeedleId(150158), ToOffset(8), 3000073)
-	m.Set(NeedleId(150162), ToOffset(8), 3000073)

-	m.AscendingVisit(func(value NeedleValue) error {
-		println("needle key:", value.Key)
+	r := rand.New(rand.NewSource(123456789))
+	r.Shuffle(len(keys), func(i, j int) { keys[i], keys[j] = keys[j], keys[i] })
+
+	cs := newCompactMapSegment(0)
+	for _, k := range keys {
+		_, _ = cs.set(k, types.Uint32ToOffset(123), 456)
+	}
+	if got, want := cs.len(), SegmentChunkSize; got != want {
+		t.Errorf("expected size %d, got %d", want, got)
+	}
+	for i := 1; i < cs.len(); i++ {
+		if ka, kb := cs.list[i-1].key, cs.list[i].key; ka >= kb {
+			t.Errorf("found out of order entries at (%d, %d) = (%d, %d)", i-1, i, ka, kb)
+		}
+	}
+}
+
+func TestSegmentGet(t *testing.T) {
+	testSegment := &CompactMapSegment{
+		list: []CompactNeedleValue{
+			CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
+			CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
+			CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
+		},
+		firstKey: 10,
+		lastKey:  30,
+	}
+
+	testCases := []struct {
+		name      string
+		key       types.NeedleId
+		wantValue *CompactNeedleValue
+		wantFound bool
+	}{
+		{
+			name:      "invalid key",
+			key:       99,
+			wantValue: nil,
+			wantFound: false,
+		},
+		{
+			name:      "key #1",
+			key:       10,
+			wantValue: &testSegment.list[0],
+			wantFound: true,
+		},
+		{
+			name:      "key #2",
+			key:       20,
+			wantValue: &testSegment.list[1],
+			wantFound: true,
+		},
+		{
+			name:      "key #3",
+			key:       30,
+			wantValue: &testSegment.list[2],
+			wantFound: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			value, found := testSegment.get(tc.key)
+			if got, want := value, tc.wantValue; got != want {
+				t.Errorf("got %v, want %v", got, want)
+			}
+			if got, want := found, tc.wantFound; got != want {
+				t.Errorf("got %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+func TestSegmentDelete(t *testing.T) {
+	testSegment := &CompactMapSegment{
+		list: []CompactNeedleValue{
+			CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
+			CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
+			CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
+			CompactNeedleValue{key: 40, offset: OffsetToCompact(types.Uint32ToOffset(600)), size: 400},
+		},
+		firstKey: 10,
+		lastKey:  40,
+	}
+
+	testDeletes := []struct {
+		name string
+		key  types.NeedleId
+		want types.Size
+	}{
+		{
+			name: "invalid key",
+			key:  99,
+			want: 0,
+		},
+		{
+			name: "delete key #2",
+			key:  20,
+			want: 200,
+		},
+		{
+			name: "delete key #4",
+			key:  40,
+			want: 400,
+		},
+	}
+
+	for _, td := range testDeletes {
+		size := testSegment.delete(td.key)
+		if got, want := size, td.want; got != want {
+			t.Errorf("%s: got %v, want %v", td.name, got, want)
+		}
+	}
+
+	wantSegment := &CompactMapSegment{
+		list: []CompactNeedleValue{
+			CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
+			CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: -200},
+			CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
+			CompactNeedleValue{key: 40, offset: OffsetToCompact(types.Uint32ToOffset(600)), size: -400},
+		},
+		firstKey: 10,
+		lastKey:  40,
+	}
+	if !reflect.DeepEqual(testSegment, wantSegment) {
+		t.Errorf("got result segment %v, want %v", testSegment, wantSegment)
+	}
+}
+
+func TestSegmentForKey(t *testing.T) {
+	testMap := NewCompactMap()
+
+	tests := []struct {
+		name string
+		key  types.NeedleId
+		want *CompactMapSegment
+	}{
+		{
+			name: "first segment",
+			key:  12,
+			want: &CompactMapSegment{
+				list:     []CompactNeedleValue{},
+				chunk:    0,
+				firstKey: MaxCompactKey,
+				lastKey:  0,
+			},
+		},
+		{
+			name: "second segment, gapless",
+			key:  SegmentChunkSize + 34,
+			want: &CompactMapSegment{
+				list:     []CompactNeedleValue{},
+				chunk:    1,
+				firstKey: MaxCompactKey,
+				lastKey:  0,
+			},
+		},
+		{
+			name: "gapped segment",
+			key:  (5 * SegmentChunkSize) + 56,
+			want: &CompactMapSegment{
+				list:     []CompactNeedleValue{},
+				chunk:    5,
+				firstKey: MaxCompactKey,
+				lastKey:  0,
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			cs := testMap.segmentForKey(tc.key)
+			if !reflect.DeepEqual(cs, tc.want) {
+				t.Errorf("got segment %v, want %v", cs, tc.want)
+			}
+		})
+	}
+
+	wantMap := &CompactMap{
+		segments: map[Chunk]*CompactMapSegment{
+			0: &CompactMapSegment{
+				list:     []CompactNeedleValue{},
+				chunk:    0,
+				firstKey: MaxCompactKey,
+				lastKey:  0,
+			},
+			1: &CompactMapSegment{
+				list:     []CompactNeedleValue{},
+				chunk:    1,
+				firstKey: MaxCompactKey,
+				lastKey:  0,
+			},
+			5: &CompactMapSegment{
+				list:     []CompactNeedleValue{},
+				chunk:    5,
+				firstKey: MaxCompactKey,
+				lastKey:  0,
+			},
+		},
+	}
+	if !reflect.DeepEqual(testMap, wantMap) {
+		t.Errorf("got map %v, want %v", testMap, wantMap)
+	}
+}
+
+func TestAscendingVisit(t *testing.T) {
+	cm := NewCompactMap()
+	for _, nid := range []types.NeedleId{20, 7, 40000, 300000, 0, 100, 500, 10000, 200000} {
+		cm.Set(nid, types.Uint32ToOffset(123), 456)
+	}
+
+	got := []NeedleValue{}
+	err := cm.AscendingVisit(func(nv NeedleValue) error {
+		got = append(got, nv)
 		return nil
 	})
-}
-
-func TestIssue52(t *testing.T) {
-	m := NewCompactMap()
-	m.Set(NeedleId(10002), ToOffset(10002), 10002)
-	if element, ok := m.Get(NeedleId(10002)); ok {
-		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
+	if err != nil {
+		t.Errorf("got error %v, expected none", err)
 	}
-	m.Set(NeedleId(10001), ToOffset(10001), 10001)
-	if element, ok := m.Get(NeedleId(10002)); ok {
-		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
-	} else {
-		t.Fatal("key 10002 missing after setting 10001")
+
+	want := []NeedleValue{
+		NeedleValue{Key: 0, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 7, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 20, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 100, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 500, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 10000, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 40000, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 200000, Offset: types.Uint32ToOffset(123), Size: 456},
+		NeedleValue{Key: 300000, Offset: types.Uint32ToOffset(123), Size: 456},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got values %v, want %v", got, want)
 	}
 }

-func TestCompactMap(t *testing.T) {
-	m := NewCompactMap()
-	for i := uint32(0); i < 100*MaxSectionBucketSize; i += 2 {
-		m.Set(NeedleId(i), ToOffset(int64(i)), Size(i))
+func TestRandomInsert(t *testing.T) {
+	count := 8 * SegmentChunkSize
+	keys := []types.NeedleId{}
+	for i := 0; i < count; i++ {
+		keys = append(keys, types.NeedleId(i))
 	}

-	for i := uint32(0); i < 100*MaxSectionBucketSize; i += 37 {
-		m.Delete(NeedleId(i))
+	r := rand.New(rand.NewSource(123456789))
+	r.Shuffle(len(keys), func(i, j int) { keys[i], keys[j] = keys[j], keys[i] })
+
+	cm := NewCompactMap()
+	for _, k := range keys {
+		_, _ = cm.Set(k, types.Uint32ToOffset(123), 456)
+	}
+	if got, want := cm.Len(), count; got != want {
+		t.Errorf("expected size %d, got %d", want, got)
 	}

-	for i := uint32(0); i < 10*MaxSectionBucketSize; i += 3 {
-		m.Set(NeedleId(i), ToOffset(int64(i+11)), Size(i+5))
-	}
-
-	//	for i := uint32(0); i < 100; i++ {
-	//		if v := m.Get(Key(i)); v != nil {
-	//			glog.V(4).Infoln(i, "=", v.Key, v.Offset, v.Size)
-	//		}
-	//	}
-
-	for i := uint32(0); i < 10*MaxSectionBucketSize; i++ {
-		v, ok := m.Get(NeedleId(i))
-		if i%3 == 0 {
-			if !ok {
-				t.Fatal("key", i, "missing!")
-			}
-			if v.Size != Size(i+5) {
-				t.Fatal("key", i, "size", v.Size)
-			}
-		} else if i%37 == 0 {
-			if ok && v.Size.IsValid() {
-				t.Fatal("key", i, "should have been deleted needle value", v)
-			}
-		} else if i%2 == 0 {
-			if v.Size != Size(i) {
-				t.Fatal("key", i, "size", v.Size)
-			}
+	last := -1
+	err := cm.AscendingVisit(func(nv NeedleValue) error {
+		key := int(nv.Key)
+		if key <= last {
+			return fmt.Errorf("found out of order entries (%d vs %d)", key, last)
 		}
+		last = key
+		return nil
+	})
+	if err != nil {
+		t.Errorf("got error %v, expected none", err)
 	}

-	for i := uint32(10 * MaxSectionBucketSize); i < 100*MaxSectionBucketSize; i++ {
-		v, ok := m.Get(NeedleId(i))
-		if i%37 == 0 {
-			if ok && v.Size.IsValid() {
-				t.Fatal("key", i, "should have been deleted needle value", v)
-			}
-		} else if i%2 == 0 {
-			if v == nil {
-				t.Fatal("key", i, "missing")
-			}
-			if v.Size != Size(i) {
-				t.Fatal("key", i, "size", v.Size)
-			}
-		}
-	}
-
-}
-
-func TestOverflow(t *testing.T) {
-	cs := NewCompactSection(1)
-
-	cs.setOverflowEntry(1, ToOffset(12), 12)
-	cs.setOverflowEntry(2, ToOffset(12), 12)
-	cs.setOverflowEntry(3, ToOffset(12), 12)
-	cs.setOverflowEntry(4, ToOffset(12), 12)
-	cs.setOverflowEntry(5, ToOffset(12), 12)
-
-	if cs.overflow[2].Key != 3 {
-		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
-	}
-
-	cs.setOverflowEntry(3, ToOffset(24), 24)
-
-	if cs.overflow[2].Key != 3 {
-		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
-	}
-
-	if cs.overflow[2].Size != 24 {
-		t.Fatalf("expecting o[2] has size 24: %+v", cs.overflow[2].Size)
-	}
-
-	cs.deleteOverflowEntry(4)
-
-	if len(cs.overflow) != 5 {
-		t.Fatalf("expecting 5 entries now: %+v", cs.overflow)
-	}
-
-	x, _ := cs.findOverflowEntry(5)
-	if x.Key != 5 {
-		t.Fatalf("expecting entry 5 now: %+v", x)
-	}
-
-	for i, x := range cs.overflow {
-		println("overflow[", i, "]:", x.Key)
-	}
-	println()
-
-	cs.deleteOverflowEntry(1)
-
-	for i, x := range cs.overflow {
-		println("overflow[", i, "]:", x.Key, "size", x.Size)
-	}
-	println()
-
-	cs.setOverflowEntry(4, ToOffset(44), 44)
-	for i, x := range cs.overflow {
-		println("overflow[", i, "]:", x.Key)
-	}
-	println()
-
-	cs.setOverflowEntry(1, ToOffset(11), 11)
-
-	for i, x := range cs.overflow {
-		println("overflow[", i, "]:", x.Key)
-	}
-	println()
-
-}
-
-func TestCompactSection_Get(t *testing.T) {
-	var maps []*CompactMap
-	totalRowCount := uint64(0)
-	indexFile, ie := os.OpenFile("../../../test/data/sample.idx",
-		os.O_RDWR|os.O_RDONLY, 0644)
-	defer indexFile.Close()
-	if ie != nil {
-		log.Fatalln(ie)
-	}
-
-	m, rowCount := loadNewNeedleMap(indexFile)
-	maps = append(maps, m)
-	totalRowCount += rowCount
-	m.Set(1574318345753513987, ToOffset(10002), 10002)
-	nv, ok := m.Get(1574318345753513987)
-	if ok {
-		t.Log(uint64(nv.Key))
-	}
-
-	nv1, ok := m.Get(1574318350048481283)
-	if ok {
-		t.Error(uint64(nv1.Key))
-	}
-
-	m.Set(1574318350048481283, ToOffset(10002), 10002)
-	nv2, ok1 := m.Get(1574318350048481283)
-	if ok1 {
-		t.Log(uint64(nv2.Key))
-	}
-
-	m.Delete(nv2.Key)
-	nv3, has := m.Get(nv2.Key)
-	if has && nv3.Size > 0 {
-		t.Error(uint64(nv3.Size))
-	}
-}
-
-// Test after putting 1 ~ LookBackWindowSize*3 items in sequential order, but missing item LookBackWindowSize
-// insert the item LookBackWindowSize in the middle of the sequence
-func TestCompactSection_PutOutOfOrderItemBeyondLookBackWindow(t *testing.T) {
-	m := NewCompactMap()
-
-	// put 1 ~ 10
-	for i := 1; i <= LookBackWindowSize*3; i++ {
-		if i != LookBackWindowSize {
-			m.Set(NeedleId(i), ToOffset(int64(i)), Size(i))
-		}
-	}
-
-	m.Set(NeedleId(LookBackWindowSize), ToOffset(int64(LookBackWindowSize)), Size(LookBackWindowSize))
-
-	// check if 8 is in the right place
-	if v, ok := m.Get(NeedleId(LookBackWindowSize)); !ok || v.Offset != ToOffset(LookBackWindowSize) || v.Size != Size(LookBackWindowSize) {
-		t.Fatalf("expected to find LookBackWindowSize at offset %d with size %d, but got %v", LookBackWindowSize, LookBackWindowSize, v)
+	// Given that we've written a integer multiple of SegmentChunkSize, all
+	// segments should be fully utilized and capacity-adjusted.
+	if l, c := cm.Len(), cm.Cap(); l != c {
+		t.Errorf("map length (%d) doesn't match capacity (%d)", l, c)
 	}
 }
@@ -0,0 +1,332 @@
+package needle_map
+
+import (
+	"sort"
+	"sync"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+
+	new_map "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+)
+
+const (
+	MaxSectionBucketSize = 1024 * 8
+	LookBackWindowSize   = 1024 // how many entries to look back when inserting into a section
+)
+
+type SectionalNeedleId uint32
+
+const SectionalNeedleIdLimit = 1<<32 - 1
+
+type SectionalNeedleValue struct {
+	Key          SectionalNeedleId
+	OffsetLower  OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
+	Size         Size        `comment:"Size of the data portion"`
+	OffsetHigher OffsetHigher
+}
+
+type CompactSection struct {
+	sync.RWMutex
+	values   []SectionalNeedleValue
+	overflow Overflow
+	start    NeedleId
+	end      NeedleId
+}
+
+type Overflow []SectionalNeedleValue
+
+func NewCompactSection(start NeedleId) *CompactSection {
+	return &CompactSection{
+		values:   make([]SectionalNeedleValue, 0),
+		overflow: Overflow(make([]SectionalNeedleValue, 0)),
+		start:    start,
+	}
+}
+
+// return old entry size
+func (cs *CompactSection) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) {
+	cs.Lock()
+	defer cs.Unlock()
+
+	if key > cs.end {
+		cs.end = key
+	}
+	skey := SectionalNeedleId(key - cs.start)
+	if i := cs.binarySearchValues(skey); i >= 0 {
+		// update
+		oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.values[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size
+		cs.values[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size
+		return
+	}
+
+	var lkey SectionalNeedleId
+	if len(cs.values) > 0 {
+		lkey = cs.values[len(cs.values)-1].Key
+	}
+
+	hasAdded := false
+	switch {
+	case len(cs.values) < MaxSectionBucketSize && lkey <= skey:
+		// non-overflow insert
+		cs.values = append(cs.values, SectionalNeedleValue{
+			Key:          skey,
+			OffsetLower:  offset.OffsetLower,
+			Size:         size,
+			OffsetHigher: offset.OffsetHigher,
+		})
+		hasAdded = true
+	case len(cs.values) < MaxSectionBucketSize:
+		// still has capacity and only partially out of order
+		lookBackIndex := len(cs.values) - LookBackWindowSize
+		if lookBackIndex < 0 {
+			lookBackIndex = 0
+		}
+		if cs.values[lookBackIndex].Key <= skey {
+			for ; lookBackIndex < len(cs.values); lookBackIndex++ {
+				if cs.values[lookBackIndex].Key >= skey {
+					break
+				}
+			}
+			cs.values = append(cs.values, SectionalNeedleValue{})
+			copy(cs.values[lookBackIndex+1:], cs.values[lookBackIndex:])
+			cs.values[lookBackIndex].Key, cs.values[lookBackIndex].Size = skey, size
+			cs.values[lookBackIndex].OffsetLower, cs.values[lookBackIndex].OffsetHigher = offset.OffsetLower, offset.OffsetHigher
+			hasAdded = true
+		}
+	}
+
+	// overflow insert
+	if !hasAdded {
+		if oldValue, found := cs.findOverflowEntry(skey); found {
+			oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValue.OffsetHigher, oldValue.OffsetLower, oldValue.Size
+		}
+		cs.setOverflowEntry(skey, offset, size)
+	} else {
+		// if we maxed out our values bucket, pin its capacity to minimize memory usage
+		if len(cs.values) == MaxSectionBucketSize {
+			bucket := make([]SectionalNeedleValue, len(cs.values))
+			copy(bucket, cs.values)
+			cs.values = bucket
+		}
+	}
+
+	return
+}
+
+func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size Size) {
+	needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size, OffsetHigher: offset.OffsetHigher}
+	insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
+		return cs.overflow[i].Key >= needleValue.Key
+	})
+
+	if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key {
+		cs.overflow[insertCandidate] = needleValue
+		return
+	}
+
+	cs.overflow = append(cs.overflow, SectionalNeedleValue{})
+	copy(cs.overflow[insertCandidate+1:], cs.overflow[insertCandidate:])
+	cs.overflow[insertCandidate] = needleValue
+}
+
+func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nv SectionalNeedleValue, found bool) {
+	foundCandidate := sort.Search(len(cs.overflow), func(i int) bool {
+		return cs.overflow[i].Key >= key
+	})
+	if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key {
+		return cs.overflow[foundCandidate], true
+	}
+	return nv, false
+}
+
+func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
+	length := len(cs.overflow)
+	deleteCandidate := sort.Search(length, func(i int) bool {
+		return cs.overflow[i].Key >= key
+	})
+	if deleteCandidate != length && cs.overflow[deleteCandidate].Key == key {
+		if cs.overflow[deleteCandidate].Size.IsValid() {
+			cs.overflow[deleteCandidate].Size = -cs.overflow[deleteCandidate].Size
+		}
+	}
+}
+
+// return old entry size
+func (cs *CompactSection) Delete(key NeedleId) Size {
+	cs.Lock()
+	defer cs.Unlock()
+	ret := Size(0)
+	if key > cs.end {
+		return ret
+	}
+	skey := SectionalNeedleId(key - cs.start)
+	if i := cs.binarySearchValues(skey); i >= 0 {
+		if cs.values[i].Size > 0 && cs.values[i].Size.IsValid() {
+			ret = cs.values[i].Size
+			cs.values[i].Size = -cs.values[i].Size
+		}
+	}
+	if v, found := cs.findOverflowEntry(skey); found {
+		cs.deleteOverflowEntry(skey)
+		ret = v.Size
+	}
+	return ret
+}
+func (cs *CompactSection) Get(key NeedleId) (*new_map.NeedleValue, bool) {
+	cs.RLock()
+	defer cs.RUnlock()
+	if key > cs.end {
+		return nil, false
+	}
+	skey := SectionalNeedleId(key - cs.start)
+	if v, ok := cs.findOverflowEntry(skey); ok {
+		nv := toNeedleValue(v, cs)
+		return &nv, true
+	}
+	if i := cs.binarySearchValues(skey); i >= 0 {
+		nv := toNeedleValue(cs.values[i], cs)
+		return &nv, true
+	}
+	return nil, false
+}
+func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {
+	x := sort.Search(len(cs.values), func(i int) bool {
+		return cs.values[i].Key >= key
+	})
+	if x >= len(cs.values) {
+		return -1
+	}
+	if cs.values[x].Key > key {
+		return -2
+	}
+	return x
+}
+
+// This map assumes mostly inserting increasing keys
+// This map assumes mostly inserting increasing keys
+type CompactMap struct {
+	list []*CompactSection
+}
+
+func NewCompactMap() *CompactMap {
+	return &CompactMap{}
+}
+
+func (cm *CompactMap) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
+		// println(x, "adding to existing", len(cm.list), "sections, starting", key)
+		cs := NewCompactSection(key)
+		cm.list = append(cm.list, cs)
+		x = len(cm.list) - 1
+		//keep compact section sorted by start
+		for x >= 0 {
+			if x > 0 && cm.list[x-1].start > key {
+				cm.list[x] = cm.list[x-1]
+				// println("shift", x, "start", cs.start, "to", x-1)
+				x = x - 1
+			} else {
+				cm.list[x] = cs
+				// println("cs", x, "start", cs.start)
+				break
+			}
+		}
+	}
+	// println(key, "set to section[", x, "].start", cm.list[x].start)
+	return cm.list[x].Set(key, offset, size)
+}
+func (cm *CompactMap) Delete(key NeedleId) Size {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 {
+		return Size(0)
+	}
+	return cm.list[x].Delete(key)
+}
+func (cm *CompactMap) Get(key NeedleId) (*new_map.NeedleValue, bool) {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 {
+		return nil, false
+	}
+	return cm.list[x].Get(key)
+}
+func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
+	l, h := 0, len(cm.list)-1
+	if h < 0 {
+		return -5
+	}
+	if cm.list[h].start <= key {
+		if len(cm.list[h].values) < MaxSectionBucketSize || key <= cm.list[h].end {
+			return h
+		}
+		return -4
+	}
+	for l <= h {
+		m := (l + h) / 2
+		if key < cm.list[m].start {
+			h = m - 1
+		} else { // cm.list[m].start <= key
+			if cm.list[m+1].start <= key {
+				l = m + 1
+			} else {
+				return m
+			}
+		}
+	}
+	return -3
+}
+
+// Visit visits all entries or stop if any error when visiting
+func (cm *CompactMap) AscendingVisit(visit func(new_map.NeedleValue) error) error {
+	for _, cs := range cm.list {
+		cs.RLock()
+		var i, j int
+		for i, j = 0, 0; i < len(cs.overflow) && j < len(cs.values); {
+			if cs.overflow[i].Key < cs.values[j].Key {
+				if err := visit(toNeedleValue(cs.overflow[i], cs)); err != nil {
+					cs.RUnlock()
+					return err
+				}
+				i++
+			} else if cs.overflow[i].Key == cs.values[j].Key {
+				j++
+			} else {
+				if err := visit(toNeedleValue(cs.values[j], cs)); err != nil {
+					cs.RUnlock()
+					return err
+				}
+				j++
+			}
+		}
+		for ; i < len(cs.overflow); i++ {
+			if err := visit(toNeedleValue(cs.overflow[i], cs)); err != nil {
+				cs.RUnlock()
+				return err
+			}
+		}
+		for ; j < len(cs.values); j++ {
+			if err := visit(toNeedleValue(cs.values[j], cs)); err != nil {
+				cs.RUnlock()
+				return err
+			}
+		}
+		cs.RUnlock()
+	}
+	return nil
+}
+
+func toNeedleValue(snv SectionalNeedleValue, cs *CompactSection) new_map.NeedleValue {
+	offset := Offset{
+		OffsetHigher: snv.OffsetHigher,
+		OffsetLower:  snv.OffsetLower,
+	}
+	return new_map.NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size}
+}
+
+func toSectionalNeedleValue(nv new_map.NeedleValue, cs *CompactSection) SectionalNeedleValue {
+	return SectionalNeedleValue{
+		Key:          SectionalNeedleId(nv.Key - cs.start),
+		OffsetLower:  nv.Offset.OffsetLower,
+		Size:         nv.Size,
+		OffsetHigher: nv.Offset.OffsetHigher,
+	}
+}
@@ -0,0 +1,92 @@
+package needle_map
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+/*
+
+To see the memory usage:
+
+go test -run TestMemoryUsage
+The Alloc section shows the in-use memory increase for each iteration.
+
+go test -run TestMemoryUsage -memprofile=mem.out
+go tool pprof --alloc_space needle.test mem.out
+
+
+*/
+
+func TestMemoryUsage(t *testing.T) {
+
+	var maps []*CompactMap
+	totalRowCount := uint64(0)
+
+	startTime := time.Now()
+	for i := 0; i < 10; i++ {
+		indexFile, ie := os.OpenFile("../../../../test/data/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
+		if ie != nil {
+			log.Fatalln(ie)
+		}
+		m, rowCount := loadNewNeedleMap(indexFile)
+		maps = append(maps, m)
+		totalRowCount += rowCount
+
+		indexFile.Close()
+
+		PrintMemUsage(totalRowCount)
+		now := time.Now()
+		fmt.Printf("\tTaken = %v\n", now.Sub(startTime))
+		startTime = now
+	}
+
+}
+
+func loadNewNeedleMap(file *os.File) (*CompactMap, uint64) {
+	m := NewCompactMap()
+	bytes := make([]byte, NeedleMapEntrySize)
+	rowCount := uint64(0)
+	count, e := file.Read(bytes)
+	for count > 0 && e == nil {
+		for i := 0; i < count; i += NeedleMapEntrySize {
+			rowCount++
+			key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
+			offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
+			size := BytesToSize(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
+
+			if !offset.IsZero() {
+				m.Set(NeedleId(key), offset, size)
+			} else {
+				m.Delete(key)
+			}
+		}
+
+		count, e = file.Read(bytes)
+	}
+
+	return m, rowCount
+
+}
+
+func PrintMemUsage(totalRowCount uint64) {
+
+	runtime.GC()
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
+	fmt.Printf("Each %.02f Bytes", float64(m.Alloc)/float64(totalRowCount))
+	fmt.Printf("\tAlloc = %v MiB", bToMb(m.Alloc))
+	fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
+	fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
+	fmt.Printf("\tNumGC = %v", m.NumGC)
+}
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}
@@ -0,0 +1,243 @@
+package needle_map
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+
+	new_map "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+)
+
+func TestSnowflakeSequencer(t *testing.T) {
+	m := NewCompactMap()
+	seq, _ := sequence.NewSnowflakeSequencer("for_test", 1)
+
+	for i := 0; i < 200000; i++ {
+		id := seq.NextFileId(1)
+		oldOffset, oldSize := m.Set(NeedleId(id), ToOffset(8), 3000073)
+		if oldSize != 0 {
+			t.Errorf("id %d oldOffset %v oldSize %d", id, oldOffset, oldSize)
+		}
+	}
+
+}
+
+func TestOverflow2(t *testing.T) {
+	m := NewCompactMap()
+	_, oldSize := m.Set(NeedleId(150088), ToOffset(8), 3000073)
+	if oldSize != 0 {
+		t.Fatalf("expecting no previous data")
+	}
+	_, oldSize = m.Set(NeedleId(150088), ToOffset(8), 3000073)
+	if oldSize != 3000073 {
+		t.Fatalf("expecting previous data size is %d, not %d", 3000073, oldSize)
+	}
+	m.Set(NeedleId(150073), ToOffset(8), 3000073)
+	m.Set(NeedleId(150089), ToOffset(8), 3000073)
+	m.Set(NeedleId(150076), ToOffset(8), 3000073)
+	m.Set(NeedleId(150124), ToOffset(8), 3000073)
+	m.Set(NeedleId(150137), ToOffset(8), 3000073)
+	m.Set(NeedleId(150147), ToOffset(8), 3000073)
+	m.Set(NeedleId(150145), ToOffset(8), 3000073)
+	m.Set(NeedleId(150158), ToOffset(8), 3000073)
+	m.Set(NeedleId(150162), ToOffset(8), 3000073)
+
+	m.AscendingVisit(func(value new_map.NeedleValue) error {
+		println("needle key:", value.Key)
+		return nil
+	})
+}
+
+func TestIssue52(t *testing.T) {
+	m := NewCompactMap()
+	m.Set(NeedleId(10002), ToOffset(10002), 10002)
+	if element, ok := m.Get(NeedleId(10002)); ok {
+		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
+	}
+	m.Set(NeedleId(10001), ToOffset(10001), 10001)
+	if element, ok := m.Get(NeedleId(10002)); ok {
+		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
+	} else {
+		t.Fatal("key 10002 missing after setting 10001")
+	}
+}
+
+func TestCompactMap(t *testing.T) {
+	m := NewCompactMap()
+	for i := uint32(0); i < 100*MaxSectionBucketSize; i += 2 {
+		m.Set(NeedleId(i), ToOffset(int64(i)), Size(i))
+	}
+
+	for i := uint32(0); i < 100*MaxSectionBucketSize; i += 37 {
+		m.Delete(NeedleId(i))
+	}
+
+	for i := uint32(0); i < 10*MaxSectionBucketSize; i += 3 {
+		m.Set(NeedleId(i), ToOffset(int64(i+11)), Size(i+5))
+	}
+
+	//	for i := uint32(0); i < 100; i++ {
+	//		if v := m.Get(Key(i)); v != nil {
+	//			glog.V(4).Infoln(i, "=", v.Key, v.Offset, v.Size)
+	//		}
+	//	}
+
+	for i := uint32(0); i < 10*MaxSectionBucketSize; i++ {
+		v, ok := m.Get(NeedleId(i))
+		if i%3 == 0 {
+			if !ok {
+				t.Fatal("key", i, "missing!")
+			}
+			if v.Size != Size(i+5) {
+				t.Fatal("key", i, "size", v.Size)
+			}
+		} else if i%37 == 0 {
+			if ok && v.Size.IsValid() {
+				t.Fatal("key", i, "should have been deleted needle value", v)
+			}
+		} else if i%2 == 0 {
+			if v.Size != Size(i) {
+				t.Fatal("key", i, "size", v.Size)
+			}
+		}
+	}
+
+	for i := uint32(10 * MaxSectionBucketSize); i < 100*MaxSectionBucketSize; i++ {
+		v, ok := m.Get(NeedleId(i))
+		if i%37 == 0 {
+			if ok && v.Size.IsValid() {
+				t.Fatal("key", i, "should have been deleted needle value", v)
+			}
+		} else if i%2 == 0 {
+			if v == nil {
+				t.Fatal("key", i, "missing")
+			}
+			if v.Size != Size(i) {
+				t.Fatal("key", i, "size", v.Size)
+			}
+		}
+	}
+
+}
+
+func TestOverflow(t *testing.T) {
+	cs := NewCompactSection(1)
+
+	cs.setOverflowEntry(1, ToOffset(12), 12)
+	cs.setOverflowEntry(2, ToOffset(12), 12)
+	cs.setOverflowEntry(3, ToOffset(12), 12)
+	cs.setOverflowEntry(4, ToOffset(12), 12)
+	cs.setOverflowEntry(5, ToOffset(12), 12)
+
+	if cs.overflow[2].Key != 3 {
+		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
+	}
+
+	cs.setOverflowEntry(3, ToOffset(24), 24)
+
+	if cs.overflow[2].Key != 3 {
+		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
+	}
+
+	if cs.overflow[2].Size != 24 {
+		t.Fatalf("expecting o[2] has size 24: %+v", cs.overflow[2].Size)
+	}
+
+	cs.deleteOverflowEntry(4)
+
+	if len(cs.overflow) != 5 {
+		t.Fatalf("expecting 5 entries now: %+v", cs.overflow)
+	}
+
+	x, _ := cs.findOverflowEntry(5)
+	if x.Key != 5 {
+		t.Fatalf("expecting entry 5 now: %+v", x)
+	}
+
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key)
+	}
+	println()
+
+	cs.deleteOverflowEntry(1)
+
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key, "size", x.Size)
+	}
+	println()
+
+	cs.setOverflowEntry(4, ToOffset(44), 44)
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key)
+	}
+	println()
+
+	cs.setOverflowEntry(1, ToOffset(11), 11)
+
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key)
+	}
+	println()
+
+}
+
+func TestCompactSection_Get(t *testing.T) {
+	var maps []*CompactMap
+	totalRowCount := uint64(0)
+	indexFile, ie := os.OpenFile("../../../../test/data/sample.idx",
+		os.O_RDWR|os.O_RDONLY, 0644)
+	defer indexFile.Close()
+	if ie != nil {
+		log.Fatalln(ie)
+	}
+
+	m, rowCount := loadNewNeedleMap(indexFile)
+	maps = append(maps, m)
+	totalRowCount += rowCount
+	m.Set(1574318345753513987, ToOffset(10002), 10002)
+	nv, ok := m.Get(1574318345753513987)
+	if ok {
+		t.Log(uint64(nv.Key))
+	}
+
+	nv1, ok := m.Get(1574318350048481283)
+	if ok {
+		t.Error(uint64(nv1.Key))
+	}
+
+	m.Set(1574318350048481283, ToOffset(10002), 10002)
+	nv2, ok1 := m.Get(1574318350048481283)
+	if ok1 {
+		t.Log(uint64(nv2.Key))
+	}
+
+	m.Delete(nv2.Key)
+	nv3, has := m.Get(nv2.Key)
+	if has && nv3.Size > 0 {
+		t.Error(uint64(nv3.Size))
+	}
+}
+
+// Test after putting 1 ~ LookBackWindowSize*3 items in sequential order, but missing item LookBackWindowSize
+// insert the item LookBackWindowSize in the middle of the sequence
+func TestCompactSection_PutOutOfOrderItemBeyondLookBackWindow(t *testing.T) {
+	m := NewCompactMap()
+
+	// put 1 ~ 10
+	for i := 1; i <= LookBackWindowSize*3; i++ {
+		if i != LookBackWindowSize {
+			m.Set(NeedleId(i), ToOffset(int64(i)), Size(i))
+		}
+	}
+
+	m.Set(NeedleId(LookBackWindowSize), ToOffset(int64(LookBackWindowSize)), Size(LookBackWindowSize))
+
+	// check if 8 is in the right place
+	if v, ok := m.Get(NeedleId(LookBackWindowSize)); !ok || v.Offset != ToOffset(LookBackWindowSize) || v.Size != Size(LookBackWindowSize) {
+		t.Fatalf("expected to find LookBackWindowSize at offset %d with size %d, but got %v", LookBackWindowSize, LookBackWindowSize, v)
+	}
+}
@@ -441,7 +441,7 @@ func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle, checkCook
 			err = fmt.Errorf("volume %d is read only", i)
 			return
 		}
-		_, _, isUnchanged, err = v.writeNeedle2(n, checkCookie, fsync || s.isStopping)
+		_, _, isUnchanged, err = v.writeNeedle2(n, checkCookie, fsync && !s.isStopping)
 		return
 	}
 	glog.V(0).Infoln("volume", i, "not found!")
@@ -4,6 +4,6 @@
 package util

 const (
-	sizeLimit         = "30GB"
+	SizeLimit         = "30GB"
 	VolumeSizeLimitGB = 30
 )
@@ -4,6 +4,6 @@
 package util

 const (
-	sizeLimit         = "8000GB"
+	SizeLimit         = "8000GB"
 	VolumeSizeLimitGB = 8000
 )
@@ -1,16 +1,18 @@
-package util
+package version

 import (
 	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 const HttpStatusCancelled = 499

 var (
 	MAJOR_VERSION  = int32(3)
-	MINOR_VERSION  = int32(89)
+	MINOR_VERSION  = int32(91)
 	VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
-	VERSION        = sizeLimit + " " + VERSION_NUMBER
+	VERSION        = util.SizeLimit + " " + VERSION_NUMBER
 	COMMIT         = ""
 )

@@ -3,6 +3,7 @@ package wdclient
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"math/rand"
 	"sync"
 	"time"
@@ -208,7 +209,7 @@ func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.Server
 			Rack:          mc.rack,
 			ClientType:    mc.clientType,
 			ClientAddress: string(mc.clientHost),
-			Version:       util.Version(),
+			Version:       version.Version(),
 		}); err != nil {
 			glog.V(0).Infof("%s.%s masterClient failed to send to %s: %v", mc.FilerGroup, mc.clientType, master, err)
 			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc()
Author	SHA1	Message	Date
chrislu	c26299b05a	3.91	2025-06-15 20:42:16 -07:00
NyaMistyandGitHub	f894e7b7a5	Support filtering source disk type in volume.tier.upload (#6868 )	2025-06-15 20:30:04 -07:00
NyaMistyandGitHub	53e5c84523	Fix wrong error handling in volume.tier.upload when stream == nil but copyErr != nil (#6867 )	2025-06-15 20:28:40 -07:00
NyaMistyandGitHub	cdc543aa9e	Correctly sort in volume.list to ensure output consistency (#6866 )	2025-06-15 20:27:48 -07:00
Gerry HernandezandGitHub	e653de54b4	FUSE Mount: Fix buffer allocation during copy (#6863 ) Fix buffer allocation during FUSE copy	2025-06-13 12:27:39 -07:00
chrislu	c79e73aa2a	mount: complete fix for freebsd fix https://github.com/seaweedfs/seaweedfs/issues/6645	2025-06-12 08:19:47 -07:00
chrislu	e71d681fee	refactor	2025-06-11 20:46:13 -07:00
chrislu	7c4d98446b	refactor	2025-06-11 20:13:06 -07:00
chrislu	f27e195354	refactoring	2025-06-11 20:13:06 -07:00
Bruce ZouandGitHub	fa730abec7	fix rocksdb enumerate (#6858 )	2025-06-11 17:55:58 -07:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	f7df4856e5	chore(deps): bump github.com/cloudflare/circl from 1.3.7 to 1.6.1 (#6856 ) Bumps [github.com/cloudflare/circl](https://github.com/cloudflare/circl) from 1.3.7 to 1.6.1. - [Release notes](https://github.com/cloudflare/circl/releases) - [Commits](https://github.com/cloudflare/circl/compare/v1.3.7...v1.6.1) --- updated-dependencies: - dependency-name: github.com/cloudflare/circl dependency-version: 1.6.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2025-06-10 19:19:12 -07:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	e9c2d5d4f8	chore(deps): bump github.com/redis/go-redis/v9 from 9.8.0 to 9.10.0 (#6850 ) Bumps [github.com/redis/go-redis/v9](https://github.com/redis/go-redis) from 9.8.0 to 9.10.0. - [Release notes](https://github.com/redis/go-redis/releases) - [Changelog](https://github.com/redis/go-redis/blob/master/CHANGELOG.md) - [Commits](https://github.com/redis/go-redis/compare/v9.8.0...v9.10.0) --- updated-dependencies: - dependency-name: github.com/redis/go-redis/v9 dependency-version: 9.10.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2025-06-09 10:38:36 -07:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	6e5e6df326	chore(deps): bump github.com/aws/aws-sdk-go-v2/service/s3 from 1.80.0 to 1.80.1 (#6852 ) chore(deps): bump github.com/aws/aws-sdk-go-v2/service/s3 Bumps [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) from 1.80.0 to 1.80.1. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.80.0...service/s3/v1.80.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.80.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2025-06-09 10:38:22 -07:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	f3b1748eb7	chore(deps): bump go.etcd.io/etcd/client/v3 from 3.6.0 to 3.6.1 (#6855 ) Bumps [go.etcd.io/etcd/client/v3](https://github.com/etcd-io/etcd) from 3.6.0 to 3.6.1. - [Release notes](https://github.com/etcd-io/etcd/releases) - [Commits](https://github.com/etcd-io/etcd/compare/v3.6.0...v3.6.1) --- updated-dependencies: - dependency-name: go.etcd.io/etcd/client/v3 dependency-version: 3.6.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2025-06-09 10:37:57 -07:00
dependabot[bot]andGitHub	dcabb71a28	chore(deps): bump github.com/aws/aws-sdk-go-v2/credentials from 1.17.67 to 1.17.68 (#6853 )	2025-06-09 09:17:09 -07:00
dependabot[bot]andGitHub	1023971dbc	chore(deps): bump modernc.org/sqlite from 1.37.1 to 1.38.0 (#6854 )	2025-06-09 09:16:49 -07:00
chrislu	33ecc8442e	refactor	2025-06-08 22:11:09 -07:00
chrislu	81aeec74a4	3.90	2025-06-08 20:59:43 -07:00
chrislu	60f11f6510	add a readme file for volume needle data layout	2025-06-07 15:52:51 -07:00
chrislu	61c4f01e05	refactor	2025-06-06 08:55:32 -07:00
chrislu	ce51d60127	include new changes related to FreeBSD https://github.com/hanwen/go-fuse/commit/50f6569d1a7d3920be218c2abdbeffeb7fbc4a3c	2025-06-06 08:48:17 -07:00
chrislu	a489d99333	fix tests	2025-06-06 08:25:04 -07:00
Lisandro PinandGitHub	00c621abb8	Fix dumb typo in `08556257` (#6844 )	2025-06-06 05:59:11 -07:00
SmoothDenisandGitHub	c99530ec66	fix(filer): add missing return after KvDelete in KvPut with empty value (#6846 )	2025-06-06 05:58:03 -07:00
chrislu	7439af0eca	refactoring	2025-06-06 01:35:48 -07:00
chrislu	cc135c63f7	a bit refactoring	2025-06-06 01:26:54 -07:00
chrislu	c4695fc3b3	refactor needle write for different versions	2025-06-06 00:35:13 -07:00
Lisandro PinandGitHub	bed0a64693	New `needle_map.CompactMap()` implementation for reduced memory usage (#6842 ) * Rework `needle_map.CompactMap()` to maximize memory efficiency. * Use a memory-efficient structure for `CompactMap` needle value entries. This slightly complicates the code, but makes a massive difference in memory efficiency - preliminary results show a ~30% reduction in heap usage, with no measurable performance impact otherwise. * Clean up type for `CompactMap` chunk IDs. * Add a small comment description for `CompactMap()`. * Add the old version of `CompactMap()` for comparison purposes.	2025-06-05 14:03:29 -07:00
chrislu	d8ddc22fc2	update to latest hanwen/go-fuse/v2 https://github.com/seaweedfs/seaweedfs/issues/6645#issuecomment-2933832235 fix https://github.com/seaweedfs/seaweedfs/issues/6645	2025-06-05 09:20:14 -07:00
chrislu	35f0daa198	the isFsync parameter is essentially IsAsyncWrite and it needs to be turned off if s.isStopping https://github.com/seaweedfs/seaweedfs/commit/d8c574a5ef1a811f9a0d447097d9edfcc0c1d84c#r159132764	2025-06-05 00:19:10 -07:00
chrislu	2f3de5e199	fix build	2025-06-03 22:50:45 -07:00
chrislu	bd4891a117	change version directory	2025-06-03 22:46:10 -07:00
chrislu	7039d5003c	gorocksdb 1.10.1 ~ rocksdb 10.2.1	2025-06-03 22:46:10 -07:00