fix build error

fix https://github.com/chrislusf/seaweedfs/issues/1923

README.md

@@ -42,6 +42,7 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
 - [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 
 Table of Contents

docker/compose/local-s3tests-compose.yml

@@ -38,7 +38,7 @@ services:
       S3TEST_CONF: "s3tests.conf"
       NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
       NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
     depends_on:
       - master
       - volume

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.10
@@ -88,7 +88,7 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
 	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
-	golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
 	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
 	google.golang.org/api v0.26.0

go.sum

@@ -959,6 +959,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0=
 golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

k8s/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.34"
-version: 2.34
+appVersion: "2.35"
+version: 2.35

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  # imageTag: "2.34" - started using {.Chart.appVersion}
+  # imageTag: "2.35" - started using {.Chart.appVersion}
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

weed/command/filer_cat.go

@@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 
 		filerCat.filerClient = client
 
-		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
 
 	})
 

weed/filer/filechunk_manifest.go

@@ -91,10 +91,10 @@ func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string,
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
 		return nil, err
 	}
-	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0)
+	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, false, 0, 0)
 }
 
-func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
+func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, isCheck bool, offset int64, size int) ([]byte, error) {
 
 	var err error
 	var buffer bytes.Buffer

weed/filer/filer_on_meta_event.go

@@ -52,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
 
 func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
 	var buf bytes.Buffer
-	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
+	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
 	if err != nil {
 		return nil, err
 	}

weed/filer/read_write.go

@@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 		return err
 	}
 
-	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
 
 }
 

weed/filer/stream.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"bytes"
 	"fmt"
+	"golang.org/x/sync/errgroup"
 	"io"
 	"math"
 	"strings"
@@ -13,7 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
-func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
+func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error {
 
 	glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
@@ -33,11 +34,29 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
 		fileId2Url[chunkView.FileId] = urlStrings
 	}
 
+	if isCheck {
+		// Pre-check all chunkViews urls
+		gErr := new(errgroup.Group)
+		CheckAllChunkViews(chunkViews, &fileId2Url, gErr)
+		if err := gErr.Wait(); err != nil {
+			glog.Errorf("check all chunks: %v", err)
+			return fmt.Errorf("check all chunks: %v", err)
+		}
+		return nil
+	}
+
 	for _, chunkView := range chunkViews {
 
 		urlStrings := fileId2Url[chunkView.FileId]
-
-		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		data, err := retriedFetchChunkData(
+			urlStrings,
+			chunkView.CipherKey,
+			chunkView.IsGzipped,
+			chunkView.IsFullChunk(),
+			false,
+			chunkView.Offset,
+			int(chunkView.Size),
+		)
 		if err != nil {
 			glog.Errorf("read chunk: %v", err)
 			return fmt.Errorf("read chunk: %v", err)
@@ -54,6 +73,24 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
 
 }
 
+func CheckAllChunkViews(chunkViews []*ChunkView, fileId2Url *map[string][]string, gErr *errgroup.Group) {
+	for _, chunkView := range chunkViews {
+		urlStrings := (*fileId2Url)[chunkView.FileId]
+		glog.V(9).Infof("Check chunk: %+v\n url: %v", chunkView, urlStrings)
+		gErr.Go(func() error {
+			_, err := retriedFetchChunkData(
+				urlStrings,
+				chunkView.CipherKey,
+				chunkView.IsGzipped,
+				chunkView.IsFullChunk(),
+				true,
+				chunkView.Offset,
+				int(chunkView.Size))
+			return err
+		})
+	}
+}
+
 // ----------------  ReadAllReader ----------------------------------
 
 func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {
@@ -73,7 +110,7 @@ func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk)
 			return nil, err
 		}
 
-		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), false, chunkView.Offset, int(chunkView.Size))
 		if err != nil {
 			return nil, err
 		}

weed/filesys/file.go

@@ -280,6 +280,9 @@ func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, er
 		return entry, err
 	}
 	if entry != nil {
+		if entry.Attributes == nil {
+			entry.Attributes = &filer_pb.FuseAttributes{}
+		}
 		file.setEntry(entry)
 	} else {
 		glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err)

weed/pb/filer_pb/filer.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: filer.proto
 

weed/pb/filer_pb/filer_client.go

@@ -182,6 +182,26 @@ func Exists(filerClient FilerClient, parentDirectoryPath string, entryName strin
 	return
 }
 
+func Touch(filerClient FilerClient, parentDirectoryPath string, entryName string, entry *Entry) (err error) {
+
+	return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {
+
+		request := &UpdateEntryRequest{
+			Directory: parentDirectoryPath,
+			Entry:     entry,
+		}
+
+		glog.V(4).Infof("touch entry %v/%v: %v", parentDirectoryPath, entryName, request)
+		if err := UpdateEntry(client, request); err != nil {
+			glog.V(0).Infof("touch exists entry %v: %v", request, err)
+			return fmt.Errorf("touch exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
+		}
+
+		return nil
+	})
+
+}
+
 func Mkdir(filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
 	return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {
 

weed/pb/iam_pb/iam.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: iam.proto
 

weed/pb/master_pb/master.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: master.proto
 

weed/pb/messaging_pb/messaging.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: messaging.proto
 

weed/pb/volume_server.proto

@@ -52,6 +52,11 @@ service VolumeServer {
     rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
     }
 
+    rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
+    }
+    rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
+    }
+
     rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
     }
     rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
@@ -253,6 +258,25 @@ message CopyFileResponse {
     bytes file_content = 1;
 }
 
+message ReadNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleBlobResponse {
+    bytes needle_blob = 1;
+}
+
+message WriteNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int32 size = 3;
+    bytes needle_blob = 4;
+}
+message WriteNeedleBlobResponse {
+}
+
 message VolumeTailSenderRequest {
     uint32 volume_id = 1;
     uint64 since_ns = 2;

weed/s3api/filer_util.go

@@ -79,6 +79,12 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD
 
 }
 
+func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {
+
+	return filer_pb.Touch(s3a, parentDirectoryPath, entryName, entry)
+
+}
+
 func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
 	fullPath := util.NewFullPath(parentDirectoryPath, entryName)
 	return filer_pb.GetEntry(s3a, fullPath)

weed/s3api/s3api_object_copy_handlers.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -25,6 +26,26 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
+
+	if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) {
+		fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
+		dir, name := fullPath.DirAndName()
+		entry, err := s3a.getEntry(dir, name)
+		if err != nil {
+			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		}
+		entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
+		err = s3a.touch(dir, name, entry)
+		if err != nil {
+			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		}
+		writeSuccessResponseXML(w, encodeResponse(CopyObjectResult{
+			ETag:         fmt.Sprintf("%x", entry.Attributes.Md5),
+			LastModified: time.Now().UTC(),
+		}))
+		return
+	}
+
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
 		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
@@ -32,7 +53,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	if srcBucket == dstBucket && srcObject == dstObject {
-		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		writeErrorResponse(w, s3err.ErrInvalidCopyDest, r.URL)
 		return
 	}
 
@@ -147,3 +168,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseXML(w, encodeResponse(response))
 
 }
+
+func isReplace(r *http.Request) bool {
+	return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE"
+}

weed/s3api/s3api_object_handlers.go

@@ -311,7 +311,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 	}
 	defer util.CloseResponse(resp)
 
-	if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
+	if resp.ContentLength == -1 || resp.StatusCode == 404 {
 		if r.Method != "DELETE" {
 			writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
 			return
@@ -326,7 +326,11 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
 	for k, v := range proxyResponse.Header {
 		w.Header()[k] = v
 	}
-	w.WriteHeader(proxyResponse.StatusCode)
+	if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
+		w.WriteHeader(http.StatusPartialContent)
+	} else {
+		w.WriteHeader(proxyResponse.StatusCode)
+	}
 	io.Copy(w, proxyResponse.Body)
 }
 

weed/server/common.go

@@ -234,12 +234,12 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file
 	}
 }
 
-func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64, httpStatusCode int) error) {
+func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
 	rangeReq := r.Header.Get("Range")
 
 	if rangeReq == "" {
 		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
-		if err := writeFn(w, 0, totalSize, 0); err != nil {
+		if err := writeFn(w, 0, totalSize); err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
@@ -279,7 +279,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
 		w.Header().Set("Content-Range", ra.contentRange(totalSize))
 
-		err = writeFn(w, ra.start, ra.length, http.StatusPartialContent)
+		err = writeFn(w, ra.start, ra.length)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -307,7 +307,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 				pw.CloseWithError(e)
 				return
 			}
-			if e = writeFn(part, ra.start, ra.length, 0); e != nil {
+			if e = writeFn(part, ra.start, ra.length); e != nil {
 				pw.CloseWithError(e)
 				return
 			}

weed/server/filer_server_handlers_read.go

@@ -131,6 +131,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 
 	if r.Method == "HEAD" {
 		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
+		processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
+			return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, true)
+		})
 		return
 	}
 
@@ -150,10 +153,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		}
 	}
 
-	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
-		if httpStatusCode != 0 {
-			w.WriteHeader(httpStatusCode)
-		}
+	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
 		if offset+size <= int64(len(entry.Content)) {
 			_, err := writer.Write(entry.Content[offset : offset+size])
 			if err != nil {
@@ -161,7 +161,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 			}
 			return err
 		}
-		return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size)
+		return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, false)
 	})
 
 }

weed/server/filer_server_handlers_write_autochunk.go

@@ -212,7 +212,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 		entry.Extended = make(map[string][]byte)
 	}
 
-	fs.saveAmzMetaData(r, entry)
+	SaveAmzMetaData(r, entry.Extended, false)
 
 	for k, v := range r.Header {
 		if len(v) > 0 && strings.HasPrefix(k, needle.PairNamePrefix) {
@@ -383,17 +383,24 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http
 	return filerResult, replyerr
 }
 
-func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
+func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte) {
+
+	metadata = make(map[string][]byte)
+	if !isReplace {
+		for k, v := range existing {
+			metadata[k] = v
+		}
+	}
 
 	if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
-		entry.Extended[xhttp.AmzStorageClass] = []byte(sc)
+		metadata[xhttp.AmzStorageClass] = []byte(sc)
 	}
 
 	if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
 		for _, v := range strings.Split(tags, "&") {
 			tag := strings.Split(v, "=")
 			if len(tag) == 2 {
-				entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
+				metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
 			}
 		}
 	}
@@ -401,8 +408,11 @@ func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
 	for header, values := range r.Header {
 		if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
 			for _, value := range values {
-				entry.Extended[header] = []byte(value)
+				metadata[header] = []byte(value)
 			}
 		}
 	}
+
+	return
+
 }

weed/server/volume_grpc_read_write.go

@@ -0,0 +1,38 @@
+package weed_server
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+)
+
+func (vs *VolumeServer) ReadNeedleBlob(ctx context.Context, req *volume_server_pb.ReadNeedleBlobRequest) (resp *volume_server_pb.ReadNeedleBlobResponse, err error) {
+	resp = &volume_server_pb.ReadNeedleBlobResponse{}
+	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
+	if v == nil {
+		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
+	}
+
+	resp.NeedleBlob, err = v.ReadNeedleBlob(req.Offset, types.Size(req.Size))
+	if err != nil {
+		return nil, fmt.Errorf("read needle blob offset %d size %d: %v", req.Offset, req.Size, err)
+	}
+
+	return resp, nil
+}
+
+func (vs *VolumeServer) WriteNeedleBlob(ctx context.Context, req *volume_server_pb.WriteNeedleBlobRequest) (resp *volume_server_pb.WriteNeedleBlobResponse, err error) {
+	resp = &volume_server_pb.WriteNeedleBlobResponse{}
+	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
+	if v == nil {
+		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
+	}
+
+	if err = v.WriteNeedleBlob(types.NeedleId(req.NeedleId), req.NeedleBlob, types.Size(req.Size)); err != nil {
+		return nil, fmt.Errorf("write blob needle %d size %d: %v", req.NeedleId, req.Size, err)
+	}
+
+	return resp, nil
+}

weed/server/volume_server_handlers_read.go

@@ -27,7 +27,7 @@ var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
 
 func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 
-	// println(r.Method + " " + r.URL.Path)
+	glog.V(9).Info(r.Method + " " + r.URL.Path + " " + r.Header.Get("Range"))
 
 	stats.VolumeServerRequestCounter.WithLabelValues("get").Inc()
 	start := time.Now()
@@ -261,13 +261,10 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re
 		return nil
 	}
 
-	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
+	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
 		if _, e = rs.Seek(offset, 0); e != nil {
 			return e
 		}
-		if httpStatusCode != 0 {
-			w.WriteHeader(httpStatusCode)
-		}
 		_, e = io.CopyN(writer, rs, size)
 		return e
 	})

weed/shell/command_fs_cat.go

@@ -52,7 +52,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write
 			return err
 		}
 
-		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
 
 	})
 

weed/shell/command_fs_mv.go

@@ -36,6 +36,10 @@ func (c *commandFsMv) Help() string {
 
 func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if len(args) != 2 {
+		return fmt.Errorf("need to have 2 arguments")
+	}
+
 	sourcePath, err := commandEnv.parseUrl(args[0])
 	if err != nil {
 		return err

weed/shell/command_volume_balance_test.go

@@ -180,4 +180,4 @@ func TestBalance(t *testing.T) {
 		t.Errorf("balance: %v", err)
 	}
 
-}
+}

weed/shell/command_volume_check_disk.go

@@ -0,0 +1,243 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
+	"io"
+	"math"
+	"sort"
+)
+
+func init() {
+	Commands = append(Commands, &commandVolumeCheckDisk{})
+}
+
+type commandVolumeCheckDisk struct {
+	env *CommandEnv
+}
+
+func (c *commandVolumeCheckDisk) Name() string {
+	return "volume.check.disk"
+}
+
+func (c *commandVolumeCheckDisk) Help() string {
+	return `check all replicated volumes to find and fix inconsistencies
+
+	How it works:
+	
+	find all volumes that are replicated
+	  for each volume id, if there are more than 2 replicas, find one pair with the largest 2 in file count.
+      for the pair volume A and B
+        append entries in A and not in B to B
+        append entries in B and not in A to A
+
+`
+}
+
+func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if err = commandEnv.confirmIsLocked(); err != nil {
+		return
+	}
+
+	fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	verbose := fsckCommand.Bool("v", false, "verbose mode")
+	applyChanges := fsckCommand.Bool("force", false, "apply the fix")
+	if err = fsckCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	c.env = commandEnv
+
+	// collect topology information
+	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	if err != nil {
+		return err
+	}
+	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
+
+	// pick 1 pairs of volume replica
+	fileCount := func(replica *VolumeReplica) uint64 {
+		return replica.info.FileCount - replica.info.DeleteCount
+	}
+	aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb()
+	defer aDB.Close()
+	defer bDB.Close()
+
+	for _, replicas := range volumeReplicas {
+		sort.Slice(replicas, func(i, j int) bool {
+			return fileCount(replicas[i]) > fileCount(replicas[j])
+		})
+		for len(replicas) >= 2 {
+			a, b := replicas[0], replicas[1]
+			if fileCount(a) == fileCount(b) {
+				replicas = replicas[1:]
+				continue
+			}
+			if a.info.ReadOnly || b.info.ReadOnly {
+				fmt.Fprintf(writer, "skipping readonly volume %d on %s and %s\n", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
+				continue
+			}
+
+			// reset index db
+			aDB.Close()
+			bDB.Close()
+			aDB, bDB = needle_map.NewMemDb(), needle_map.NewMemDb()
+
+			// read index db
+			if err := c.readIndexDatabase(aDB, a.info.Collection, a.info.Id, a.location.dataNode.Id, *verbose, writer); err != nil {
+				return err
+			}
+			if err := c.readIndexDatabase(bDB, b.info.Collection, b.info.Id, b.location.dataNode.Id, *verbose, writer); err != nil {
+				return err
+			}
+
+			// find and make up the differnces
+			if err := c.doVolumeCheckDisk(aDB, bDB, a, b, *verbose, writer, *applyChanges); err != nil {
+				return err
+			}
+			if err := c.doVolumeCheckDisk(bDB, aDB, b, a, *verbose, writer, *applyChanges); err != nil {
+				return err
+			}
+			replicas = replicas[1:]
+		}
+	}
+
+	return nil
+}
+
+func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_map.MemDb, source, target *VolumeReplica, verbose bool, writer io.Writer, applyChanges bool) error {
+
+	// find missing keys
+	// hash join, can be more efficient
+	var missingNeedles []needle_map.NeedleValue
+	var counter int
+	subtrahend.AscendingVisit(func(value needle_map.NeedleValue) error {
+		counter++
+		if _, found := minuend.Get(value.Key); !found {
+			missingNeedles = append(missingNeedles, value)
+		}
+		return nil
+	})
+
+	fmt.Fprintf(writer, "%s has %d entries, %s missed %d entries\n", source.location.dataNode.Id, counter, target.location.dataNode.Id, len(missingNeedles))
+
+	for _, needleValue := range missingNeedles {
+
+		needleBlob, err := c.readSourceNeedleBlob(source.location.dataNode.Id, source.info.Id, needleValue)
+		if err != nil {
+			return err
+		}
+
+		if !applyChanges {
+			continue
+		}
+
+		if verbose {
+			fmt.Fprintf(writer, "read %d,%x %s => %s \n", source.info.Id, needleValue.Key, source.location.dataNode.Id, target.location.dataNode.Id)
+		}
+
+		if err := c.writeNeedleBlobToTarget(target.location.dataNode.Id, source.info.Id, needleValue, needleBlob); err != nil {
+			return err
+		}
+
+	}
+
+
+	return nil
+}
+
+func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue) (needleBlob []byte, err error) {
+
+	err = operation.WithVolumeServerClient(sourceVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		resp, err := client.ReadNeedleBlob(context.Background(), &volume_server_pb.ReadNeedleBlobRequest{
+			VolumeId: volumeId,
+			NeedleId: uint64(needleValue.Key),
+			Offset: needleValue.Offset.ToActualOffset(),
+			Size: int32(needleValue.Size),
+		})
+		if err != nil {
+			return err
+		}
+		needleBlob = resp.NeedleBlob
+		return nil
+	})
+	return
+}
+
+func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue, needleBlob []byte) error {
+
+	return operation.WithVolumeServerClient(targetVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
+			VolumeId: volumeId,
+			NeedleId: uint64(needleValue.Key),
+			Size: int32(needleValue.Size),
+			NeedleBlob: needleBlob,
+		})
+		return err
+	})
+
+}
+
+func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection string, volumeId uint32, volumeServer string, verbose bool, writer io.Writer) error {
+
+	var buf bytes.Buffer
+	if err := c.copyVolumeIndexFile(collection, volumeId, volumeServer, &buf, verbose, writer); err != nil {
+		return err
+	}
+
+	if verbose {
+		fmt.Fprintf(writer, "load collection %s volume %d index size %d from %s ...\n", collection, volumeId, buf.Len(), volumeServer)
+	}
+
+	return db.LoadFromReaderAt(bytes.NewReader(buf.Bytes()))
+
+}
+
+func (c *commandVolumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint32, volumeServer string, buf *bytes.Buffer, verbose bool, writer io.Writer) error {
+
+	return operation.WithVolumeServerClient(volumeServer, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+
+		ext := ".idx"
+
+		copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
+			VolumeId:                 volumeId,
+			Ext:                      ".idx",
+			CompactionRevision:       math.MaxUint32,
+			StopOffset:               math.MaxInt64,
+			Collection:               collection,
+			IsEcVolume:               false,
+			IgnoreSourceFileNotFound: false,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
+		}
+
+		err = writeToBuffer(copyFileClient, buf)
+		if err != nil {
+			return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, volumeServer, err)
+		}
+
+		return nil
+
+	})
+}
+
+func writeToBuffer(client volume_server_pb.VolumeServer_CopyFileClient, buf *bytes.Buffer) error {
+	for {
+		resp, receiveErr := client.Recv()
+		if receiveErr == io.EOF {
+			break
+		}
+		if receiveErr != nil {
+			return fmt.Errorf("receiving: %v", receiveErr)
+		}
+		buf.Write(resp.FileContent)
+	}
+	return nil
+}

weed/shell/command_volume_list_test.go

@@ -67,7 +67,7 @@ func parseOutput(output string) *master_pb.TopologyInfo {
 				maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:]
 				maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr)
 				disk = &master_pb.DiskInfo{
-					Type: diskType,
+					Type:           diskType,
 					MaxVolumeCount: uint64(maxVolumeCount),
 				}
 				dn.DiskInfos[types.ToDiskType(diskType).String()] = disk

weed/shell/shell_liner.go

@@ -19,6 +19,10 @@ var (
 
 func RunShell(options ShellOptions) {
 
+	sort.Slice(Commands, func(i, j int) bool {
+		return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
+	})
+
 	line = liner.NewLiner()
 	defer line.Close()
 
@@ -96,9 +100,6 @@ func printGenericHelp() {
 `
 	fmt.Print(msg)
 
-	sort.Slice(Commands, func(i, j int) bool {
-		return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
-	})
 	for _, c := range Commands {
 		helpTexts := strings.SplitN(c.Help(), "\n", 2)
 		fmt.Printf("  %-30s\t# %s \n", c.Name(), helpTexts[0])
@@ -114,10 +115,6 @@ func printHelp(cmds []string) {
 	} else {
 		cmd := strings.ToLower(args[0])
 
-		sort.Slice(Commands, func(i, j int) bool {
-			return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
-		})
-
 		for _, c := range Commands {
 			if c.Name() == cmd {
 				fmt.Printf("  %s\t# %s\n", c.Name(), c.Help())

weed/storage/needle/needle_read_write.go

@@ -3,13 +3,12 @@ package needle
 import (
 	"errors"
 	"fmt"
-	"io"
-	"math"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"io"
+	"math"
 )
 
 const (
@@ -156,6 +155,35 @@ func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset u
 	return offset, size, actualSize, err
 }
 
+func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size, appendAtNs uint64, version Version) (offset uint64, err error) {
+
+	if end, _, e := w.GetStat(); e == nil {
+		defer func(w backend.BackendStorageFile, off int64) {
+			if err != nil {
+				if te := w.Truncate(end); te != nil {
+					glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
+				}
+			}
+		}(w, end)
+		offset = uint64(end)
+	} else {
+		err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
+		return
+	}
+
+	if version == Version3 {
+		tsOffset := NeedleHeaderSize + size + NeedleChecksumSize
+		util.Uint64toBytes(dataSlice[tsOffset : tsOffset+TimestampSize], appendAtNs)
+	}
+
+	if err == nil {
+		_, err = w.WriteAt(dataSlice, int64(offset))
+	}
+
+	return
+
+}
+
 func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error) {
 
 	dataSize := GetActualSize(size, version)

weed/storage/needle_map/memdb.go

@@ -2,6 +2,7 @@ package needle_map
 
 import (
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/syndtr/goleveldb/leveldb"
@@ -104,7 +105,13 @@ func (cm *MemDb) LoadFromIdx(idxName string) (ret error) {
 	}
 	defer idxFile.Close()
 
-	return idx.WalkIndexFile(idxFile, func(key NeedleId, offset Offset, size Size) error {
+	return cm.LoadFromReaderAt(idxFile)
+
+}
+
+func (cm *MemDb) LoadFromReaderAt(readerAt io.ReaderAt) (ret error) {
+
+	return idx.WalkIndexFile(readerAt, func(key NeedleId, offset Offset, size Size) error {
 		if offset.IsZero() || size.IsDeleted() {
 			return cm.Delete(key)
 		}

weed/storage/volume_read.go

@@ -0,0 +1,131 @@
+package storage
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage/backend"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+)
+
+// read fills in Needle content by looking up n.Id from NeedleMapper
+func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption) (int, error) {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+
+	nv, ok := v.nm.Get(n.Id)
+	if !ok || nv.Offset.IsZero() {
+		return -1, ErrorNotFound
+	}
+	readSize := nv.Size
+	if readSize.IsDeleted() {
+		if readOption != nil && readOption.ReadDeleted && readSize != TombstoneFileSize {
+			glog.V(3).Infof("reading deleted %s", n.String())
+			readSize = -readSize
+		} else {
+			return -1, ErrorDeleted
+		}
+	}
+	if readSize == 0 {
+		return 0, nil
+	}
+	err := n.ReadData(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version())
+	if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
+		err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version())
+	}
+	v.checkReadWriteError(err)
+	if err != nil {
+		return 0, err
+	}
+	bytesRead := len(n.Data)
+	if !n.HasTtl() {
+		return bytesRead, nil
+	}
+	ttlMinutes := n.Ttl.Minutes()
+	if ttlMinutes == 0 {
+		return bytesRead, nil
+	}
+	if !n.HasLastModifiedDate() {
+		return bytesRead, nil
+	}
+	if time.Now().Before(time.Unix(0, int64(n.AppendAtNs)).Add(time.Duration(ttlMinutes) * time.Minute)) {
+		return bytesRead, nil
+	}
+	return -1, ErrorNotFound
+}
+
+// read fills in Needle content by looking up n.Id from NeedleMapper
+func (v *Volume) ReadNeedleBlob(offset int64, size Size) ([]byte, error) {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+
+	return needle.ReadNeedleBlob(v.DataBackend, offset, size, v.Version())
+}
+
+type VolumeFileScanner interface {
+	VisitSuperBlock(super_block.SuperBlock) error
+	ReadNeedleBody() bool
+	VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error
+}
+
+func ScanVolumeFile(dirname string, collection string, id needle.VolumeId,
+	needleMapKind NeedleMapKind,
+	volumeFileScanner VolumeFileScanner) (err error) {
+	var v *Volume
+	if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil {
+		return fmt.Errorf("failed to load volume %d: %v", id, err)
+	}
+	if err = volumeFileScanner.VisitSuperBlock(v.SuperBlock); err != nil {
+		return fmt.Errorf("failed to process volume %d super block: %v", id, err)
+	}
+	defer v.Close()
+
+	version := v.Version()
+
+	offset := int64(v.SuperBlock.BlockSize())
+
+	return ScanVolumeFileFrom(version, v.DataBackend, offset, volumeFileScanner)
+}
+
+func ScanVolumeFileFrom(version needle.Version, datBackend backend.BackendStorageFile, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
+	n, nh, rest, e := needle.ReadNeedleHeader(datBackend, version, offset)
+	if e != nil {
+		if e == io.EOF {
+			return nil
+		}
+		return fmt.Errorf("cannot read %s at offset %d: %v", datBackend.Name(), offset, e)
+	}
+	for n != nil {
+		var needleBody []byte
+		if volumeFileScanner.ReadNeedleBody() {
+			// println("needle", n.Id.String(), "offset", offset, "size", n.Size, "rest", rest)
+			if needleBody, err = n.ReadNeedleBody(datBackend, version, offset+NeedleHeaderSize, rest); err != nil {
+				glog.V(0).Infof("cannot read needle head [%d, %d) body [%d, %d) body length %d: %v", offset, offset+NeedleHeaderSize, offset+NeedleHeaderSize, offset+NeedleHeaderSize+rest, rest, err)
+				// err = fmt.Errorf("cannot read needle body: %v", err)
+				// return
+			}
+		}
+		err := volumeFileScanner.VisitNeedle(n, offset, nh, needleBody)
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			glog.V(0).Infof("visit needle error: %v", err)
+			return fmt.Errorf("visit needle error: %v", err)
+		}
+		offset += NeedleHeaderSize + rest
+		glog.V(4).Infof("==> new entry offset %d", offset)
+		if n, nh, rest, err = needle.ReadNeedleHeader(datBackend, version, offset); err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return fmt.Errorf("cannot read needle header at offset %d: %v", offset, err)
+		}
+		glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
+	}
+	return nil
+}

weed/storage/volume_write.go

@@ -4,14 +4,12 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
-	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
@@ -146,7 +144,8 @@ func (v *Volume) doWriteRequest(n *needle.Needle) (offset uint64, size Size, isU
 			return
 		}
 		if existingNeedle.Cookie != n.Cookie {
-			glog.V(0).Infof("write cookie mismatch: existing %x, new %x", existingNeedle.Cookie, n.Cookie)
+			glog.V(0).Infof("write cookie mismatch: existing %s, new %s",
+				needle.NewFileIdFromNeedle(v.Id, existingNeedle), needle.NewFileIdFromNeedle(v.Id, n))
 			err = fmt.Errorf("mismatching cookie %x", n.Cookie)
 			return
 		}
@@ -226,52 +225,6 @@ func (v *Volume) doDeleteRequest(n *needle.Needle) (Size, error) {
 	return 0, nil
 }
 
-// read fills in Needle content by looking up n.Id from NeedleMapper
-func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption) (int, error) {
-	v.dataFileAccessLock.RLock()
-	defer v.dataFileAccessLock.RUnlock()
-
-	nv, ok := v.nm.Get(n.Id)
-	if !ok || nv.Offset.IsZero() {
-		return -1, ErrorNotFound
-	}
-	readSize := nv.Size
-	if readSize.IsDeleted() {
-		if readOption != nil && readOption.ReadDeleted && readSize != TombstoneFileSize {
-			glog.V(3).Infof("reading deleted %s", n.String())
-			readSize = -readSize
-		} else {
-			return -1, ErrorDeleted
-		}
-	}
-	if readSize == 0 {
-		return 0, nil
-	}
-	err := n.ReadData(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version())
-	if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
-		err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version())
-	}
-	v.checkReadWriteError(err)
-	if err != nil {
-		return 0, err
-	}
-	bytesRead := len(n.Data)
-	if !n.HasTtl() {
-		return bytesRead, nil
-	}
-	ttlMinutes := n.Ttl.Minutes()
-	if ttlMinutes == 0 {
-		return bytesRead, nil
-	}
-	if !n.HasLastModifiedDate() {
-		return bytesRead, nil
-	}
-	if time.Now().Before(time.Unix(0, int64(n.AppendAtNs)).Add(time.Duration(ttlMinutes) * time.Minute)) {
-		return bytesRead, nil
-	}
-	return -1, ErrorNotFound
-}
-
 func (v *Volume) startWorker() {
 	go func() {
 		chanClosed := false
@@ -347,66 +300,28 @@ func (v *Volume) startWorker() {
 	}()
 }
 
-type VolumeFileScanner interface {
-	VisitSuperBlock(super_block.SuperBlock) error
-	ReadNeedleBody() bool
-	VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error
-}
+func (v *Volume) WriteNeedleBlob(needleId NeedleId, needleBlob []byte, size Size) error {
 
-func ScanVolumeFile(dirname string, collection string, id needle.VolumeId,
-	needleMapKind NeedleMapKind,
-	volumeFileScanner VolumeFileScanner) (err error) {
-	var v *Volume
-	if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil {
-		return fmt.Errorf("failed to load volume %d: %v", id, err)
-	}
-	if err = volumeFileScanner.VisitSuperBlock(v.SuperBlock); err != nil {
-		return fmt.Errorf("failed to process volume %d super block: %v", id, err)
-	}
-	defer v.Close()
-
-	version := v.Version()
+	v.dataFileAccessLock.Lock()
+	defer v.dataFileAccessLock.Unlock()
 
-	offset := int64(v.SuperBlock.BlockSize())
+	if MaxPossibleVolumeSize < v.nm.ContentSize()+uint64(len(needleBlob)) {
+		return fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.nm.ContentSize())
+	}
 
-	return ScanVolumeFileFrom(version, v.DataBackend, offset, volumeFileScanner)
-}
+	appendAtNs := uint64(time.Now().UnixNano())
+	offset, err := needle.WriteNeedleBlob(v.DataBackend, needleBlob, size, appendAtNs, v.Version())
 
-func ScanVolumeFileFrom(version needle.Version, datBackend backend.BackendStorageFile, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
-	n, nh, rest, e := needle.ReadNeedleHeader(datBackend, version, offset)
-	if e != nil {
-		if e == io.EOF {
-			return nil
-		}
-		return fmt.Errorf("cannot read %s at offset %d: %v", datBackend.Name(), offset, e)
+	v.checkReadWriteError(err)
+	if err != nil {
+		return err
 	}
-	for n != nil {
-		var needleBody []byte
-		if volumeFileScanner.ReadNeedleBody() {
-			// println("needle", n.Id.String(), "offset", offset, "size", n.Size, "rest", rest)
-			if needleBody, err = n.ReadNeedleBody(datBackend, version, offset+NeedleHeaderSize, rest); err != nil {
-				glog.V(0).Infof("cannot read needle head [%d, %d) body [%d, %d) body length %d: %v", offset, offset+NeedleHeaderSize, offset+NeedleHeaderSize, offset+NeedleHeaderSize+rest, rest, err)
-				// err = fmt.Errorf("cannot read needle body: %v", err)
-				// return
-			}
-		}
-		err := volumeFileScanner.VisitNeedle(n, offset, nh, needleBody)
-		if err == io.EOF {
-			return nil
-		}
-		if err != nil {
-			glog.V(0).Infof("visit needle error: %v", err)
-			return fmt.Errorf("visit needle error: %v", err)
-		}
-		offset += NeedleHeaderSize + rest
-		glog.V(4).Infof("==> new entry offset %d", offset)
-		if n, nh, rest, err = needle.ReadNeedleHeader(datBackend, version, offset); err != nil {
-			if err == io.EOF {
-				return nil
-			}
-			return fmt.Errorf("cannot read needle header at offset %d: %v", offset, err)
-		}
-		glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
+	v.lastAppendAtNs = appendAtNs
+
+	// add to needle map
+	if err = v.nm.Put(needleId, ToOffset(int64(offset)), size); err != nil {
+		glog.V(4).Infof("failed to put in needle map %d: %v", needleId, err)
 	}
-	return nil
+
+	return err
 }

weed/util/constants.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 34)
+	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 35)
 	COMMIT  = ""
 )