2.09

Chart Fix

.github/workflows/release.yml

@@ -3,8 +3,6 @@ name: Release
 on:
   push:
     branches: [ master ]
-  pull_request:
-    branches: [ master ]
 
 jobs:
 
@@ -16,14 +14,28 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
 
-#    - name: Go Release Binaries
-#      uses: wangyoucao577/go-release-action@v1.8
-#      with:
-#        github_token: ${{ secrets.GITHUB_TOKEN }}
-#        goos: linux # default is 
-#        goarch: amd64 # default is 
-#        build_flags: -tags 5BytesOffset # optional, default is 
-#        ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
-#        # Where to run `go build .`
-#        project_path: weed
-#        binary_name: weed-large-disk
+    - name: Go Release Binaries
+      uses: wangyoucao577/go-release-action@feature/customize-target-release
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        goos: linux # default is 
+        goarch: amd64 # default is 
+        release_tag: dev
+        overwrite: true 
+        build_flags: -tags 5BytesOffset # optional, default is 
+        ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+        # Where to run `go build .`
+        project_path: weed
+        binary_name: weed-large-disk
+    - name: Go Release Binaries
+      uses: wangyoucao577/go-release-action@feature/customize-target-release
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        goos: linux # default is 
+        goarch: amd64 # default is 
+        release_tag: dev
+        overwrite: true 
+        ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+        # Where to run `go build .`
+        project_path: weed
+        binary_name: weed

README.md

@@ -62,7 +62,8 @@ Table of Contents
 * [Features](#features)
     * [Additional Features](#additional-features)
     * [Filer Features](#filer-features)
-* [Example Usage](#example-usage)
+* [Quick Start](#quick-start)
+* [Example: Using Seaweed Object Store](#example-Using-Seaweed-Object-Store)
 * [Architecture](#architecture)
 * [Compared to Other File Systems](#compared-to-other-file-systems)
     * [Compared to HDFS](#compared-to-hdfs)
@@ -141,7 +142,13 @@ On top of the object store, optional [Filer] can support directories and POSIX a
 
 [Back to TOC](#table-of-contents)
 
-## Example Usage ##
+## Quick Start ##
+* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
+* Run `weed server -dir=. -s3` to start one master, one volume server, one filer, and one S3 gateway, with data stored in current directory.
+
+You can also start each module individually. See `weed -h`, or `weed <command> -h` for help.
+
+## Example: Using Seaweed Object Store ##
 
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.
 Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.

k8s/seaweedfs/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 2.08
+version: 2.09

k8s/seaweedfs/templates/filer-statefulset.yaml

@@ -101,7 +101,7 @@ spec:
               -port={{ .Values.filer.port }} \
               {{- if .Values.filer.metricsPort }}
               -metricsPort {{ .Values.filer.metricsPort }} \
-              {{- end }}}
+              {{- end }}
               {{- if .Values.filer.redirectOnRead }}
               -redirectOnRead \
               {{- end }}

k8s/seaweedfs/templates/s3-deployment.yaml

@@ -73,7 +73,7 @@ spec:
               -port={{ .Values.s3.port }} \
               {{- if .Values.s3.metricsPort }}
               -metricsPort {{ .Values.s3.metricsPort }} \
-              {{- end }}}
+              {{- end }}
               {{- if .Values.global.enableSecurity }}
               -cert.file=/usr/local/share/ca-certificates/client/tls.crt \
               -key.file=/usr/local/share/ca-certificates/client/tls.key \

k8s/seaweedfs/templates/s3-service.yaml

@@ -14,10 +14,10 @@ spec:
     protocol: TCP
 {{- if .Values.s3.metricsPort }}
   - name: "swfs-s3-metrics"
-    port: {{ .Values.filer.s3 }}
+    port: {{ .Values.s3.metricsPort }}
     targetPort: {{ .Values.s3.metricsPort }}
     protocol: TCP
-{{- end }}}
+{{- end }}
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: s3

k8s/seaweedfs/templates/s3-servicemonitor.yaml

@@ -15,4 +15,4 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: s3
-{{- end }}}
+{{- end }}

k8s/seaweedfs/templates/seaweedfs-grafana-dashboard.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.global.monitoring.enabled }}
+{{- $files := .Files.Glob "dashboards/*.json" }}
+{{- if $files }}
+apiVersion: v1
+kind: ConfigMapList
+items:
+{{- range $path, $fileContents := $files }}
+{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }}
+- apiVersion: v1
+  kind: ConfigMap
+  metadata:
+    name: {{ printf "%s" $dashboardName | lower | replace "_" "-" }}
+    namespace: {{ $.Release.Namespace }}
+    labels:
+     grafana_dashboard: "1"
+  data:
+    {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }}
+{{- end }}
+{{- end }}
+{{- end }}

k8s/seaweedfs/templates/volume-service.yaml

@@ -22,7 +22,7 @@ spec:
     port: {{ .Values.volume.metricsPort }}
     targetPort: {{ .Values.volume.metricsPort }}
     protocol: TCP
-{{- end }}}
+{{- end }}
 selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume

k8s/seaweedfs/templates/volume-servicemonitor.yaml

@@ -15,4 +15,4 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume
-{{- end }}}
+{{- end }}

k8s/seaweedfs/templates/volume-statefulset.yaml

@@ -78,7 +78,7 @@ spec:
                 -port={{ .Values.volume.port }} \
                 {{- if .Values.volume.metricsPort }}
                 -metricsPort {{ .Values.volume.metricsPort }} \
-                {{- end }}}
+                {{- end }}
                 -dir={{ .Values.volume.dir }} \
                 -max={{ .Values.volume.maxVolumes }} \
                 {{- if .Values.volume.rack }}

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "2.08"
+  imageTag: "2.09"
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.2</version>
+    <version>1.5.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.2</version>
+    <version>1.5.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.2</version>
+    <version>1.5.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -65,6 +65,12 @@ public class SeaweedFileSystem extends FileSystem {
 
     }
 
+    @Override
+    public void close() throws IOException {
+        this.seaweedFileSystemStore.close();
+        super.close();
+    }
+
     @Override
     public FSDataInputStream open(Path path, int bufferSize) throws IOException {
 

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -36,6 +36,14 @@ public class SeaweedFileSystemStore {
         this.conf = conf;
     }
 
+    public void close() {
+        try {
+            this.filerGrpcClient.shutdown();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+
     public static String getParentDirectory(Path path) {
         return path.isRoot() ? "/" : path.getParent().toUri().getPath();
     }

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -65,6 +65,12 @@ public class SeaweedFileSystem extends FileSystem {
 
     }
 
+    @Override
+    public void close() throws IOException {
+        this.seaweedFileSystemStore.close();
+        super.close();
+    }
+
     @Override
     public FSDataInputStream open(Path path, int bufferSize) throws IOException {
 

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -36,6 +36,14 @@ public class SeaweedFileSystemStore {
         this.conf = conf;
     }
 
+    public void close() {
+        try {
+            this.filerGrpcClient.shutdown();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+
     public static String getParentDirectory(Path path) {
         return path.isRoot() ? "/" : path.getParent().toUri().getPath();
     }

weed/filesys/dir.go

@@ -362,6 +362,14 @@ func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
 
 	// then, delete meta cache and fsNode cache
 	dir.wfs.metaCache.DeleteEntry(context.Background(), filePath)
+
+	// clear entry inside the file
+	fsNode := dir.wfs.fsNodeCache.GetFsNode(filePath)
+	if fsNode != nil {
+		if file, ok := fsNode.(*File); ok {
+			file.clearEntry()
+		}
+	}
 	dir.wfs.fsNodeCache.DeleteFsNode(filePath)
 
 	// remove current file handle if any

weed/filesys/dirty_page.go

@@ -15,10 +15,10 @@ type ContinuousDirtyPages struct {
 	intervals              *ContinuousIntervals
 	f                      *File
 	writeWaitGroup         sync.WaitGroup
+	chunkAddLock           sync.Mutex
 	chunkSaveErrChan       chan error
 	chunkSaveErrChanClosed bool
 	lastErr                error
-	lock                   sync.Mutex
 	collection             string
 	replication            string
 }
@@ -117,6 +117,8 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
 		}
 		chunk.Mtime = mtime
 		pages.collection, pages.replication = collection, replication
+		pages.chunkAddLock.Lock()
+		defer pages.chunkAddLock.Unlock()
 		pages.f.addChunks([]*filer_pb.FileChunk{chunk})
 		glog.V(3).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size)
 	}

weed/filesys/file.go

@@ -144,7 +144,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 				}
 			}
 			file.entry.Chunks = chunks
-			file.entryViewCache = nil
+			file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), chunks)
 			file.reader = nil
 			file.wfs.deleteFileChunks(truncatedChunks)
 		}
@@ -282,15 +282,37 @@ func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, er
 	return entry, nil
 }
 
+func lessThan(a, b *filer_pb.FileChunk) bool {
+	if a.Mtime == b.Mtime {
+		return a.Fid.FileKey < b.Fid.FileKey
+	}
+	return a.Mtime < b.Mtime
+}
+
 func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
 
-	sort.Slice(chunks, func(i, j int) bool {
-		if chunks[i].Mtime == chunks[j].Mtime {
-			return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey
+	// find the earliest incoming chunk
+	newChunks := chunks
+	earliestChunk := newChunks[0]
+	for i:=1;i<len(newChunks);i++{
+		if lessThan(earliestChunk, newChunks[i]) {
+			earliestChunk = newChunks[i]
 		}
-		return chunks[i].Mtime < chunks[j].Mtime
+	}
+
+	// pick out-of-order chunks from existing chunks
+	for _, chunk := range file.entry.Chunks {
+		if lessThan(earliestChunk, chunk) {
+			chunks = append(chunks, chunk)
+		}
+	}
+
+	// sort incoming chunks
+	sort.Slice(chunks, func(i, j int) bool {
+		return lessThan(chunks[i], chunks[j])
 	})
 
+	// add to entry view cache
 	for _, chunk := range chunks {
 		file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk)
 	}
@@ -299,7 +321,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
 
 	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(file.entry.Chunks), len(chunks))
 
-	file.entry.Chunks = append(file.entry.Chunks, chunks...)
+	file.entry.Chunks = append(file.entry.Chunks, newChunks...)
 }
 
 func (file *File) setEntry(entry *filer_pb.Entry) {

weed/filesys/filehandle.go

@@ -183,16 +183,18 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 	}
 
 	if fh.f.isOpen == 0 {
+
 		if err := fh.doFlush(ctx, req.Header); err != nil {
 			glog.Errorf("Release doFlush %s: %v", fh.f.Name, err)
 		}
-		fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
-	}
 
-	// stop the goroutine
-	if !fh.dirtyPages.chunkSaveErrChanClosed {
-		fh.dirtyPages.chunkSaveErrChanClosed = true
-		close(fh.dirtyPages.chunkSaveErrChan)
+		// stop the goroutine
+		if !fh.dirtyPages.chunkSaveErrChanClosed {
+			fh.dirtyPages.chunkSaveErrChanClosed = true
+			close(fh.dirtyPages.chunkSaveErrChan)
+		}
+
+		fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
 	}
 
 	return nil
@@ -262,7 +264,6 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
 			glog.V(0).Infof("MaybeManifestize: %v", manifestErr)
 		}
 		fh.f.entry.Chunks = append(chunks, manifestChunks...)
-		fh.f.entryViewCache = nil
 
 		fh.f.wfs.mapPbIdFromLocalToFiler(request.Entry)
 		defer fh.f.wfs.mapPbIdFromFilerToLocal(request.Entry)

weed/operation/upload_content.go

@@ -81,14 +81,11 @@ func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader,
 	if ok {
 		data = bytesReader.Bytes
 	} else {
-		buf := bytebufferpool.Get()
-		_, err = buf.ReadFrom(reader)
-		defer bytebufferpool.Put(buf)
+		data, err = ioutil.ReadAll(reader)
 		if err != nil {
 			err = fmt.Errorf("read input: %v", err)
 			return
 		}
-		data = buf.Bytes()
 	}
 	uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
 	return uploadResult, uploadErr, data

weed/s3api/s3api_object_handlers.go

@@ -266,7 +266,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	resp, postErr := client.Do(proxyReq)
 
-	if resp.ContentLength == -1 && !strings.HasSuffix(destUrl, "/") {
+	if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
 		writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
 		return
 	}

weed/server/filer_server_handlers_read.go

@@ -94,10 +94,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		}
 	}
 
+	// print out the header from extended properties
+	for k, v := range entry.Extended {
+		w.Header().Set(k, string(v))
+	}
+
 	//set tag count
 	if r.Method == "GET" {
 		tagCount := 0
-		for k, _ := range entry.Extended {
+		for k := range entry.Extended {
 			if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") {
 				tagCount++
 			}

weed/server/filer_server_handlers_write_autochunk.go

@@ -21,6 +21,7 @@ import (
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/stats"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -177,8 +178,18 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 		Size: chunkOffset,
 	}
 
+	if entry.Extended == nil {
+		entry.Extended = make(map[string][]byte)
+	}
+
 	fs.saveAmzMetaData(r, entry)
 
+	for k, v := range r.Header {
+		if len(v) > 0 && strings.HasPrefix(k, needle.PairNamePrefix) {
+			entry.Extended[k] = []byte(v[0])
+		}
+	}
+
 	if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
 		fs.filer.DeleteChunks(entry.Chunks)
 		replyerr = dbErr
@@ -314,10 +325,6 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http
 
 func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
 
-	if entry.Extended == nil {
-		entry.Extended = make(map[string][]byte)
-	}
-
 	if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
 		entry.Extended[xhttp.AmzStorageClass] = []byte(sc)
 	}

weed/server/volume_grpc_client_to_master.go

@@ -203,6 +203,7 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi
 			}
 		case <-volumeTickChan:
 			glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port)
+			vs.store.MaybeAdjustVolumeMax()
 			if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
 				glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
 				return "", err

weed/storage/disk_location.go

@@ -17,11 +17,12 @@ import (
 )
 
 type DiskLocation struct {
-	Directory           string
-	MaxVolumeCount      int
-	MinFreeSpacePercent float32
-	volumes             map[needle.VolumeId]*Volume
-	volumesLock         sync.RWMutex
+	Directory              string
+	MaxVolumeCount         int
+	OriginalMaxVolumeCount int
+	MinFreeSpacePercent    float32
+	volumes                map[needle.VolumeId]*Volume
+	volumesLock            sync.RWMutex
 
 	// erasure coding
 	ecVolumes     map[needle.VolumeId]*erasure_coding.EcVolume
@@ -31,7 +32,7 @@ type DiskLocation struct {
 }
 
 func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32) *DiskLocation {
-	location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount, MinFreeSpacePercent: minFreeSpacePercent}
+	location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount, OriginalMaxVolumeCount: maxVolumeCount, MinFreeSpacePercent: minFreeSpacePercent}
 	location.volumes = make(map[needle.VolumeId]*Volume)
 	location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
 	go location.CheckDiskSpace()

weed/storage/store.go

@@ -461,7 +461,8 @@ func (s *Store) GetVolumeSizeLimit() uint64 {
 func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
 	volumeSizeLimit := s.GetVolumeSizeLimit()
 	for _, diskLocation := range s.Locations {
-		if diskLocation.MaxVolumeCount == 0 {
+		if diskLocation.OriginalMaxVolumeCount == 0 {
+			currentMaxVolumeCount := diskLocation.MaxVolumeCount
 			diskStatus := stats.NewDiskStatus(diskLocation.Directory)
 			unusedSpace := diskLocation.UnUsedSpace(volumeSizeLimit)
 			unclaimedSpaces := int64(diskStatus.Free) - int64(unusedSpace)
@@ -473,7 +474,7 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
 			diskLocation.MaxVolumeCount = maxVolumeCount
 			glog.V(0).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
 				diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024)
-			hasChanges = true
+			hasChanges = hasChanges || currentMaxVolumeCount != diskLocation.MaxVolumeCount
 		}
 	}
 	return