diff --git a/weed/command/filer.go b/weed/command/filer.go index a15075abb..a0a90b251 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -145,6 +145,7 @@ func init() { filerS3Options.metricsHttpIp = cmdFiler.Flag.String("s3.metricsIp", "", "metrics listen ip. If empty, default to same as -s3.ip.bind option.") cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.") filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + filerS3Options.autoCreateBucket = cmdFiler.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only") filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-.sock") filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file") filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate") diff --git a/weed/command/mini.go b/weed/command/mini.go index a02d40e7d..3dd5853ff 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -389,6 +389,7 @@ var ( miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file") miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3") miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + miniS3AutoCreateBucket = cmdMini.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only") miniBucket = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.") miniTableBucket = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.") ) @@ -518,6 +519,7 @@ func initMiniS3Flags() { miniS3Options.iamConfig = miniIamConfig miniS3Options.auditLogConfig = cmdMini.Flag.String("s3.auditLogConfig", "", "path to the audit log config file") miniS3Options.allowDeleteBucketNotEmpty = miniS3AllowDeleteBucketNotEmpty + miniS3Options.autoCreateBucket = miniS3AutoCreateBucket miniS3Options.externalUrl = cmdMini.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.") miniS3Options.defaultFileMode = cmdMini.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666") miniS3Options.cacheSizeMB = cmdMini.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)") diff --git a/weed/command/s3.go b/weed/command/s3.go index e57d33de8..6d1037071 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -59,6 +59,7 @@ type S3Options struct { metricsHttpPort *int metricsHttpIp *string allowDeleteBucketNotEmpty *bool + autoCreateBucket *bool auditLogConfig *string localFilerSocket *string dataCenter *string @@ -104,6 +105,7 @@ func init() { s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") cmdS3.Flag.Bool("allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.") s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + s3StandaloneOptions.autoCreateBucket = cmdS3.Flag.Bool("autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only") s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path") s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-.sock") s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 120, "connection idle seconds") @@ -351,6 +353,7 @@ func (s3opt *S3Options) startS3Server() bool { BucketsPath: filerBucketsPath, GrpcDialOption: grpcDialOption, AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty, + AutoCreateBucket: *s3opt.autoCreateBucket, LocalFilerSocket: localFilerSocket, DataCenter: *s3opt.dataCenter, FilerGroup: filerGroup, diff --git a/weed/command/server.go b/weed/command/server.go index e89d26682..a40f13175 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -175,6 +175,7 @@ func init() { s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file") cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.") s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + s3Options.autoCreateBucket = cmdServer.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only") s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-.sock") s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.") s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 120, "connection idle seconds") diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 2f4cae39d..5e261ef93 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -589,6 +589,9 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC // ErrAutoCreatePermissionDenied is returned when a user lacks permission to auto-create buckets var ErrAutoCreatePermissionDenied = errors.New("permission denied - requires Admin permission") +// ErrAutoCreateDisabled is returned when bucket auto-creation is disabled by configuration +var ErrAutoCreateDisabled = errors.New("bucket auto-creation is disabled") + // ErrInvalidBucketName is returned when a bucket name doesn't meet S3 naming requirements var ErrInvalidBucketName = errors.New("invalid bucket name") @@ -690,6 +693,10 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error { return fmt.Errorf("auto-create bucket %s: %w", bucket, errors.Join(ErrInvalidBucketName, err)) } + if !s3a.option.AutoCreateBucket { + return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreateDisabled) + } + // Check if user has admin permissions if !s3a.isUserAdmin(r) { return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreatePermissionDenied) @@ -758,7 +765,9 @@ func (s3a *S3ApiServer) handleAutoCreateBucket(w http.ResponseWriter, r *http.Re if err := s3a.autoCreateBucket(r, bucket); err != nil { glog.Warningf("%s: %v", handlerName, err) // Check for specific errors to return appropriate S3 error codes - if errors.Is(err, ErrInvalidBucketName) { + if errors.Is(err, ErrAutoCreateDisabled) { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) + } else if errors.Is(err, ErrInvalidBucketName) { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidBucketName) } else if errors.Is(err, ErrAutoCreatePermissionDenied) { s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied) diff --git a/weed/s3api/s3api_bucket_handlers_misc_test.go b/weed/s3api/s3api_bucket_handlers_misc_test.go index b25d8720e..8f8081e90 100644 --- a/weed/s3api/s3api_bucket_handlers_misc_test.go +++ b/weed/s3api/s3api_bucket_handlers_misc_test.go @@ -221,3 +221,67 @@ func TestGetBucketLogging(t *testing.T) { t.Fatalf("missing xmlns: %s", got) } } + +func TestHandleAutoCreateBucketDisabled(t *testing.T) { + s3a := &S3ApiServer{option: &S3ApiServerOption{}} + req := newBucketRequest(http.MethodPut, "test-bucket", "", "") + rec := httptest.NewRecorder() + + if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") { + t.Fatal("expected auto-create to be rejected") + } + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound) + } + if !strings.Contains(rec.Body.String(), "NoSuchBucket") { + t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String()) + } +} + +func TestHandleAutoCreateBucketNonAdmin(t *testing.T) { + s3a := &S3ApiServer{option: &S3ApiServerOption{AutoCreateBucket: true}} + req := newBucketRequest(http.MethodPut, "test-bucket", "", "") + rec := httptest.NewRecorder() + + if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") { + t.Fatal("expected auto-create to be rejected") + } + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusForbidden) + } +} + +func TestUploadMissingBucketAutoCreateDisabled(t *testing.T) { + cases := []struct { + name string + method string + object string + handler func(*S3ApiServer, http.ResponseWriter, *http.Request) + }{ + {"put object", http.MethodPut, "/key", (*S3ApiServer).PutObjectHandler}, + {"put directory marker", http.MethodPut, "/dir/", (*S3ApiServer).PutObjectHandler}, + {"new multipart upload", http.MethodPost, "/key", (*S3ApiServer).NewMultipartUploadHandler}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s3a := &S3ApiServer{ + option: &S3ApiServerOption{}, + iam: &IdentityAccessManagement{}, + bucketConfigCache: NewBucketConfigCache(time.Minute), + } + s3a.bucketConfigCache.SetNegativeCache("missing") + req := httptest.NewRequest(tc.method, "/missing"+tc.object, strings.NewReader("")) + req = mux.SetURLVars(req, map[string]string{"bucket": "missing", "object": tc.object}) + rec := httptest.NewRecorder() + + tc.handler(s3a, rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound) + } + if !strings.Contains(rec.Body.String(), "NoSuchBucket") { + t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String()) + } + }) + } +} diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 95d6d9ed0..5f8652ac7 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -131,6 +131,17 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) objectContentType := r.Header.Get("Content-Type") if strings.HasSuffix(object, "/") && r.ContentLength <= 1024 { + // The filer mkdir below would implicitly create a missing bucket, so gate it + // on the same auto-create policy as regular uploads. + if err := s3a.checkBucket(r, bucket); err == s3err.ErrNoSuchBucket { + if !s3a.handleAutoCreateBucket(w, r, bucket, "PutObjectHandler") { + return + } + } else if err != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, err) + return + } + // Split the object into directory path and name objectWithoutSlash := strings.TrimSuffix(object, "/") dirName := path.Dir(objectWithoutSlash) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 7b2e6c746..7b09e0855 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -52,6 +52,7 @@ type S3ApiServerOption struct { BucketsPath string GrpcDialOption grpc.DialOption AllowDeleteBucketNotEmpty bool + AutoCreateBucket bool // create the bucket on upload if it does not exist LocalFilerSocket string DataCenter string FilerGroup string