diff --git a/weed/remote_storage/azure/azure_storage_client.go b/weed/remote_storage/azure/azure_storage_client.go index c27f5846f..38970e850 100644 --- a/weed/remote_storage/azure/azure_storage_client.go +++ b/weed/remote_storage/azure/azure_storage_client.go @@ -324,7 +324,7 @@ func (az *azureRemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.Remot } data = make([]byte, size) - _, err = blobClient.DownloadBuffer(context.Background(), data, &blob.DownloadBufferOptions{ + n, err := blobClient.DownloadBuffer(context.Background(), data, &blob.DownloadBufferOptions{ Range: blob.HTTPRange{ Offset: offset, Count: size, @@ -335,6 +335,11 @@ func (az *azureRemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.Remot if err != nil { return nil, fmt.Errorf("failed to download file %s%s: %w", loc.Bucket, loc.Path, err) } + // Pre-sized buffer: a short read stays zero-padded. Reject it rather than + // cache corrupt content. + if n != size { + return nil, fmt.Errorf("short read from %s%s at offset %d: got %d bytes, want %d", loc.Bucket, loc.Path, offset, n, size) + } return data, nil } diff --git a/weed/remote_storage/s3/s3_storage_client.go b/weed/remote_storage/s3/s3_storage_client.go index ec0ab0f06..9c7755019 100644 --- a/weed/remote_storage/s3/s3_storage_client.go +++ b/weed/remote_storage/s3/s3_storage_client.go @@ -243,7 +243,7 @@ func (s *s3RemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.RemoteSto dataSlice := make([]byte, int(size)) writerAt := aws.NewWriteAtBuffer(dataSlice) - _, err = downloader.Download(writerAt, &s3.GetObjectInput{ + n, err := downloader.Download(writerAt, &s3.GetObjectInput{ Bucket: aws.String(loc.Bucket), Key: aws.String(loc.Path[1:]), Range: aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)), @@ -251,6 +251,12 @@ func (s *s3RemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.RemoteSto if err != nil { return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err) } + // The buffer is pre-sized to size, so a short read leaves the tail + // zero-padded and would be cached as valid-looking but corrupt content. + // Reject it instead. + if n != size { + return nil, fmt.Errorf("short read from %s%s at offset %d: got %d bytes, want %d", loc.Bucket, loc.Path, offset, n, size) + } return writerAt.Bytes(), nil } diff --git a/weed/server/volume_grpc_remote.go b/weed/server/volume_grpc_remote.go index ad6eebe21..1be5fc3eb 100644 --- a/weed/server/volume_grpc_remote.go +++ b/weed/server/volume_grpc_remote.go @@ -229,6 +229,12 @@ func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_ser if readRemoteErr != nil { return nil, fmt.Errorf("read from remote %+v: %w", remoteStorageLocation, readRemoteErr) } + // The chunk is recorded with the requested size, so a short read would be + // cached as a full-size chunk with a zero-padded or truncated tail. Fail + // loudly instead of persisting silently corrupt content. + if int64(len(data)) != req.Size { + return nil, fmt.Errorf("read from remote %+v: got %d bytes, want %d", remoteStorageLocation, len(data), req.Size) + } var wg sync.WaitGroup wg.Add(1) diff --git a/weed/shell/command_remote_cache.go b/weed/shell/command_remote_cache.go index b035c69bb..575c9a771 100644 --- a/weed/shell/command_remote_cache.go +++ b/weed/shell/command_remote_cache.go @@ -273,7 +273,7 @@ func (c *commandRemoteCache) doComprehensiveSync(commandEnv *CommandEnv, writer Attributes: &filer_pb.FuseAttributes{ FileSize: uint64(remoteEntry.RemoteSize), Mtime: remoteEntry.RemoteMtime, - FileMode: uint32(0644), + FileMode: remoteEntryFileMode(isDirectory), }, RemoteEntry: remoteEntry, }, diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go index dc9cb9c68..fb10d9bfc 100644 --- a/weed/shell/command_remote_meta_sync.go +++ b/weed/shell/command_remote_meta_sync.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "io" + "os" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -121,6 +122,20 @@ If entry.RemoteEntry == nil, this is a new local change and should not be overwr the remote version is updated, need to pull meta } */ + +// remoteEntryFileMode returns the POSIX mode for an entry synced from a remote +// whose listing carries no mode. It matches what SeaweedFS S3 assigns native +// objects (files 0660) so a mounted bucket matches the source, deriving the +// directory mode with the same 0111 traversal mask the filer uses for +// auto-created parents (0660 -> 0771). +func remoteEntryFileMode(isDirectory bool) uint32 { + mode := uint32(0660) + if isDirectory { + mode = uint32(os.ModeDir) | mode | 0111 + } + return mode +} + func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf) error { // visit remote storage @@ -159,7 +174,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util Attributes: &filer_pb.FuseAttributes{ FileSize: uint64(remoteEntry.RemoteSize), Mtime: remoteEntry.RemoteMtime, - FileMode: uint32(0644), + FileMode: remoteEntryFileMode(isDirectory), TtlSec: 0, // Remote entries should not have TTL }, RemoteEntry: remoteEntry,