minor

fix https://github.com/chrislusf/seaweedfs/issues/3166

.github/workflows/binaries_dev.yml

@@ -17,14 +17,13 @@ jobs:
     steps:
 
       - name: Delete old release assets
-        uses: mknejp/delete-release-assets@a8aaab13272b1eaac16cc46dddd3f725b97ee05a # v1
+        uses: mknejp/delete-release-assets@v1
         with:
           token: ${{ github.token }}
           tag: dev
           fail-if-no-assets: false
           assets: |
             weed-*
-          fail-if-no-release: false
 
   build_dev_linux_windows:
     permissions:

weed/command/scaffold/shell.toml

@@ -3,8 +3,6 @@ default = "c1"
 
 [cluster.c1]
 master = "localhost:9333"    # comma-separated master servers
-filer = "localhost:8888"     # filer host and port
 
 [cluster.c2]
 master = ""
-filer = ""

weed/filer/filechunks.go

@@ -3,11 +3,9 @@ package filer
 import (
 	"bytes"
 	"fmt"
-	"math"
-	"sync"
-
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"golang.org/x/exp/slices"
+	"math"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -54,11 +52,11 @@ func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
 	if len(chunks) == 1 {
 		return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag))
 	}
-	md5_digests := [][]byte{}
+	var md5Digests [][]byte
 	for _, c := range chunks {
-		md5_digests = append(md5_digests, util.Base64Md5ToBytes(c.ETag))
+		md5Digests = append(md5Digests, util.Base64Md5ToBytes(c.ETag))
 	}
-	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks))
+	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5Digests, nil)), len(chunks))
 }
 
 func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
@@ -189,12 +187,6 @@ func logPrintf(name string, visibles []VisibleInterval) {
 	*/
 }
 
-var bufPool = sync.Pool{
-	New: func() interface{} {
-		return new(VisibleInterval)
-	},
-}
-
 func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (newVisibles []VisibleInterval) {
 
 	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, 0, chunk.Size, chunk.CipherKey, chunk.IsCompressed)

weed/filer/reader_at.go

@@ -164,6 +164,10 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
 func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, nextChunkViews []*ChunkView, offset uint64) (n int, err error) {
 
 	if c.readerPattern.IsRandomMode() {
+		n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
+		if n > 0 {
+			return n, err
+		}
 		return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
 	}
 

weed/mount/filehandle.go

@@ -6,7 +6,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"golang.org/x/exp/slices"
-	"io"
 	"sync"
 )
 
@@ -24,7 +23,7 @@ type FileHandle struct {
 	dirtyMetadata  bool
 	dirtyPages     *PageWriter
 	entryViewCache []filer.VisibleInterval
-	reader         io.ReaderAt
+	reader         *filer.ChunkReadAt
 	contentType    string
 	handle         uint64
 	sync.Mutex
@@ -99,8 +98,16 @@ func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
 	fh.entryViewCache = nil
 }
 
+func (fh *FileHandle) SetReader(reader *filer.ChunkReadAt) {
+	if fh.reader != nil {
+		fh.reader.Close()
+	}
+	fh.reader = reader
+}
+
 func (fh *FileHandle) Release() {
 	fh.dirtyPages.Destroy()
+	fh.SetReader(nil)
 }
 
 func lessThan(a, b *filer_pb.FileChunk) bool {

weed/mount/filehandle_read.go

@@ -62,7 +62,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 		if chunkResolveErr != nil {
 			return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
 		}
-		fh.reader = nil
+		fh.SetReader(nil)
 	}
 
 	reader := fh.reader
@@ -74,7 +74,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 		}
 		reader = filer.NewChunkReaderAtFromClient(fh.wfs.LookupFn(), chunkViews, fh.wfs.chunkCache, fileSize)
 	}
-	fh.reader = reader
+	fh.SetReader(reader)
 
 	totalRead, err := reader.ReadAt(buff, offset)
 

weed/replication/sub/notification_gocdk_pub_sub.go

@@ -46,17 +46,17 @@ func QueueDeclareAndBind(conn *amqp.Connection, exchangeUrl string, queueUrl str
 	}
 	defer ch.Close()
 	if err := ch.ExchangeDeclare(
-		exchangeNameDLX, "fanout", false, false, false, false, nil); err != nil {
+		exchangeNameDLX, "fanout", true, false, false, false, nil); err != nil {
 		glog.Error(err)
 		return err
 	}
 	if err := ch.ExchangeDeclare(
-		exchangeName, "fanout", false, false, false, false, nil); err != nil {
+		exchangeName, "fanout", true, false, false, false, nil); err != nil {
 		glog.Error(err)
 		return err
 	}
 	if _, err := ch.QueueDeclare(
-		queueName, false, false, false, false,
+		queueName, true, false, false, false,
 		amqp.Table{"x-dead-letter-exchange": exchangeNameDLX}); err != nil {
 		glog.Error(err)
 		return err
@@ -66,7 +66,7 @@ func QueueDeclareAndBind(conn *amqp.Connection, exchangeUrl string, queueUrl str
 		return err
 	}
 	if _, err := ch.QueueDeclare(
-		queueNameDLX, false, false, false, false,
+		queueNameDLX, true, false, false, false,
 		amqp.Table{"x-dead-letter-exchange": exchangeName, "x-message-ttl": 600000}); err != nil {
 		glog.Error(err)
 		return err

weed/s3api/s3api_objects_list_handlers.go

@@ -281,7 +281,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 	request := &filer_pb.ListEntriesRequest{
 		Directory:          dir,
 		Prefix:             prefix,
-		Limit:              uint32(maxKeys + 1),
+		Limit:              uint32(maxKeys + 2), // bucket root directory needs to skip additional s3_constants.MultipartUploadsFolder folder
 		StartFromFileName:  marker,
 		InclusiveStartFrom: inclusiveStartFrom,
 	}

weed/shell/command_fs_du.go

@@ -45,7 +45,7 @@ func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer
 	blockCount, byteCount, err = duTraverseDirectory(writer, commandEnv, dir, name)
 
 	if name == "" && err == nil {
-		fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir)
+		fmt.Fprintf(writer, "block:%4d\tlogical size:%10d\t%s\n", blockCount, byteCount, dir)
 	}
 
 	return
@@ -76,7 +76,7 @@ func duTraverseDirectory(writer io.Writer, filerClient filer_pb.FilerClient, dir
 		}
 
 		if name != "" && !entry.IsDirectory {
-			fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s/%s\n", fileBlockCount, fileByteCount, dir, entry.Name)
+			fmt.Fprintf(writer, "block:%4d\tlogical size:%10d\t%s/%s\n", fileBlockCount, fileByteCount, dir, entry.Name)
 		}
 		return nil
 	})

weed/topology/volume_location_list.go

@@ -82,12 +82,13 @@ func (dnll *VolumeLocationList) Refresh(freshThreshHold int64) {
 	}
 }
 
+// Stats returns logic size and count
 func (dnll *VolumeLocationList) Stats(vid needle.VolumeId, freshThreshHold int64) (size uint64, fileCount int) {
 	for _, dnl := range dnll.list {
 		if dnl.LastSeen < freshThreshHold {
 			vinfo, err := dnl.GetVolumesById(vid)
 			if err == nil {
-				return (vinfo.Size - vinfo.DeletedByteCount) * uint64(len(dnll.list)), vinfo.FileCount - vinfo.DeleteCount
+				return (vinfo.Size - vinfo.DeletedByteCount), vinfo.FileCount - vinfo.DeleteCount
 			}
 		}
 	}