diff --git a/weed/command/filer.go b/weed/command/filer.go index c4f43dff7..a15075abb 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -80,6 +80,8 @@ type FilerOptions struct { allowedOrigins *string exposeDirectoryData *bool tusBasePath *string + tusMaxSizeMB *int + tusSessionExpiry *time.Duration s3ConfigFile *string // optional path to static S3 identity config // shutdownCtx, when non-nil, tells startFiler to gracefully shut down its // HTTP/gRPC servers once the ctx is cancelled. Used by integration tests @@ -123,6 +125,8 @@ func init() { f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins") f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI") f.tusBasePath = cmdFiler.Flag.String("tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)") + f.tusMaxSizeMB = cmdFiler.Flag.Int("tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB") + f.tusSessionExpiry = cmdFiler.Flag.Duration("tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") @@ -386,6 +390,8 @@ func (fo *FilerOptions) startFiler() { DiskType: *fo.diskType, AllowedOrigins: strings.Split(*fo.allowedOrigins, ","), TusBasePath: *fo.tusBasePath, + TusMaxSize: int64(*fo.tusMaxSizeMB) * 1024 * 1024, + TusSessionExpiry: *fo.tusSessionExpiry, CredentialManager: credentialManager, }) if nfs_err != nil { diff --git a/weed/command/mini.go b/weed/command/mini.go index 09c825b61..a7a0ba01a 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -458,6 +458,8 @@ func initMiniFilerFlags() { miniFilerOptions.allowedOrigins = cmdMini.Flag.String("filer.allowedOrigins", "*", "comma separated list of allowed origins") miniFilerOptions.exposeDirectoryData = cmdMini.Flag.Bool("filer.exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI") miniFilerOptions.tusBasePath = cmdMini.Flag.String("filer.tusBasePath", "/.tus", "TUS resumable upload endpoint base path") + miniFilerOptions.tusMaxSizeMB = cmdMini.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB") + miniFilerOptions.tusSessionExpiry = cmdMini.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration") } // initMiniVolumeFlags initializes Volume server flag options diff --git a/weed/command/server.go b/weed/command/server.go index 19360d6a8..e89d26682 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -130,6 +130,8 @@ func init() { filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") filerOptions.exposeDirectoryData = cmdServer.Flag.Bool("filer.exposeDirectoryData", true, "expose directory data via filer. If false, filer UI will be inaccessible.") filerOptions.tusBasePath = cmdServer.Flag.String("filer.tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)") + filerOptions.tusMaxSizeMB = cmdServer.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB") + filerOptions.tusSessionExpiry = cmdServer.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port") diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 9ea143228..11b1c080b 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -84,6 +84,8 @@ type FilerOption struct { AllowedOrigins []string ExposeDirectoryData bool TusBasePath string + TusMaxSize int64 + TusSessionExpiry time.Duration S3ConfigFile string // optional path to static S3 identity config file CredentialManager *credential.CredentialManager } @@ -254,6 +256,12 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) if option.TusBasePath == "" { glog.Warningf("Invalid TUS base path; TUS disabled (must not be root '/')") } else { + if option.TusMaxSize <= 0 { + option.TusMaxSize = TusDefaultMaxSize + } + if option.TusSessionExpiry <= 0 { + option.TusSessionExpiry = TusDefaultSessionExpiry + } handlePath := option.TusBasePath + "/" defaultMux.HandleFunc(handlePath, fs.filerGuard.WhiteList(requestIDMiddleware(fs.tusHandler))) // Start background cleanup of expired TUS sessions (every hour) diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.go index 623cbe6ef..5b72e45a8 100644 --- a/weed/server/filer_server_tus_handlers.go +++ b/weed/server/filer_server_tus_handlers.go @@ -175,7 +175,7 @@ func writeTusCompleteError(w http.ResponseWriter, err error) { func (fs *FilerServer) tusOptionsHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Tus-Version", TusVersion) w.Header().Set("Tus-Extension", TusExtensions) - w.Header().Set("Tus-Max-Size", strconv.FormatInt(TusMaxSize, 10)) + w.Header().Set("Tus-Max-Size", strconv.FormatInt(fs.option.TusMaxSize, 10)) w.WriteHeader(http.StatusOK) } @@ -197,7 +197,7 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request) http.Error(w, "Invalid Upload-Length", http.StatusBadRequest) return } - if uploadLength > TusMaxSize { + if uploadLength > fs.option.TusMaxSize { http.Error(w, "Upload-Length exceeds maximum", http.StatusRequestEntityTooLarge) return } diff --git a/weed/server/filer_server_tus_idor_test.go b/weed/server/filer_server_tus_idor_test.go index aa4ae5806..2a66547bd 100644 --- a/weed/server/filer_server_tus_idor_test.go +++ b/weed/server/filer_server_tus_idor_test.go @@ -82,7 +82,7 @@ func TestFilerServer_tusHandler_RejectsAliasesAndInvalidMetadata(t *testing.T) { {"empty stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "", Size: 1}}, {"root stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/", Size: 1}}, {"relative stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "buckets/secret/x.bin", Size: 1}}, - {"oversize stored size", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/buckets/secret/victim.bin", Size: TusMaxSize + 1}}, + {"oversize stored size", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/buckets/secret/victim.bin", Size: TusDefaultMaxSize + 1}}, } for _, tt := range tests { diff --git a/weed/server/filer_server_tus_jwt_test.go b/weed/server/filer_server_tus_jwt_test.go index af02a202e..c8291ed2b 100644 --- a/weed/server/filer_server_tus_jwt_test.go +++ b/weed/server/filer_server_tus_jwt_test.go @@ -48,7 +48,7 @@ func newTusTestServer(t *testing.T, sessions map[string]string) (*FilerServer, * fs := &FilerServer{ filer: newRenameTestFiler(t, store), filerGuard: security.NewGuard(nil, tusTestWriteKey, 0, tusTestReadKey, 0), - option: &FilerOption{TusBasePath: "/.tus"}, + option: &FilerOption{TusBasePath: "/.tus", TusMaxSize: TusDefaultMaxSize}, } for uploadID, targetPath := range sessions { seedTusSession(t, fs, store, TusSession{ID: uploadID, TargetPath: targetPath, Size: 1}) diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.go index 4bc838466..5c0494d44 100644 --- a/weed/server/filer_server_tus_session.go +++ b/weed/server/filer_server_tus_session.go @@ -20,12 +20,13 @@ import ( ) const ( - TusVersion = "1.0.0" - TusMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB default max size - TusUploadsFolder = ".uploads.tus" - TusInfoFileName = ".info" - TusChunkExt = ".chunk" - TusExtensions = "creation,creation-with-upload,termination" + TusVersion = "1.0.0" + TusDefaultMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB + TusDefaultSessionExpiry = 24 * time.Hour + TusUploadsFolder = ".uploads.tus" + TusInfoFileName = ".info" + TusChunkExt = ".chunk" + TusExtensions = "creation,creation-with-upload,termination" ) // ErrWormEnforced marks a TUS completion rejected because the target entry is @@ -124,7 +125,7 @@ func (fs *FilerServer) createTusSession(ctx context.Context, uploadID, targetPat Offset: 0, Metadata: metadata, CreatedAt: time.Now(), - ExpiresAt: time.Now().Add(7 * 24 * time.Hour), // 7 days default expiration + ExpiresAt: time.Now().Add(fs.option.TusSessionExpiry), Chunks: []*TusChunkInfo{}, } @@ -209,7 +210,7 @@ func (fs *FilerServer) readTusSessionInfo(ctx context.Context, uploadID string) if target == "" || target == "/" { return nil, fmt.Errorf("invalid TUS target path: %q", session.TargetPath) } - if session.Size < 0 || session.Size > TusMaxSize { + if session.Size < 0 || session.Size > fs.option.TusMaxSize { return nil, fmt.Errorf("invalid TUS upload size: %d", session.Size) } // Pin authorization and every later operation to the same canonical path.