feat(mini): add -bucket flag to pre-create an S3 bucket on startup (#9302)

* feat(mini): add -bucket flag to pre-create an S3 bucket on startup Lets users hand a pre-provisioned object store to clients/CI without a post-start `weed shell s3.bucket.create` step. The flag is a no-op when empty (default) and idempotent on subsequent starts. * mini: bound bucket-creation RPCs with a timeout off miniClientsCtx Address PR review feedback: derive the lookup/mkdir context from miniClientsCtx() so Ctrl+C cancels the bucket RPCs, and cap with a 5s timeout so a stalled filer cannot block the welcome message indefinitely. Also wrap the DoMkdir error for parity with the lookup path. * mini: fall back to S3_BUCKET env var for -bucket Mirrors the existing -s3.externalUrl / S3_EXTERNAL_URL pattern so container/Kubernetes deployments can pre-create the bucket via env without overriding the entrypoint command. * docs(readme): lead weed mini quick start with credentials + bucket Promote the one-line setup (env vars + bucket) so users get a ready-to-use S3 endpoint without hopping between sections to find credential and bucket setup. * mini: accept comma-separated -bucket list Lets a single startup pre-create multiple S3 buckets, e.g. -bucket=bucket1,bucket2 (or S3_BUCKET=bucket1,bucket2). Names are trimmed and deduped; per-bucket errors are logged and the loop continues so one bad name does not block the rest. * mini: add -tableBucket flag for pre-creating S3 Tables buckets Mirrors -bucket but creates S3 Tables (Iceberg) buckets via s3tables.Manager so users can hand the all-in-one binary a ready-to-use table catalog without a follow-up weed shell call. Comma-separated, env fallback to S3_TABLE_BUCKET, idempotent on restart, owned by the DefaultAccountID placeholder. * mini: use errors.Is for ErrNotFound check in bucket lookup Matches the rest of the codebase (~20 call sites in weed/s3api). The direct equality works today because LookupEntry returns ErrNotFound unwrapped, but errors.Is future-proofs against any future wrapping.
2026-08-16 04:06:44 +00:00 · 2026-05-02 21:02:21 -07:00
parent 1f6f473995
commit f16353de0b
3 changed files with 174 additions and 10 deletions
@@ -80,28 +80,29 @@ Table of Contents


 ## Quick Start with weed mini ##
-The easiest way to get started with SeaweedFS for development and testing:

-* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
-
-Example:
+Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:

 ```bash
-# remove quarantine on macOS
-# xattr -d com.apple.quarantine  ./weed
-
+AWS_ACCESS_KEY_ID=admin \
+AWS_SECRET_ACCESS_KEY=secret \
+S3_BUCKET=my-bucket \
 ./weed mini -dir=/data
 ```

-This single command starts a complete SeaweedFS setup with:
+That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
+
+The same command starts everything else too:
+- **S3 Endpoint**: http://localhost:8333
 - **Master UI**: http://localhost:9333
 - **Volume Server**: http://localhost:9340
 - **Filer UI**: http://localhost:8888
- **S3 Endpoint**: http://localhost:8333
 - **WebDAV**: http://localhost:7333
 - **Admin UI**: http://localhost:23646

-Perfect for development, testing, learning SeaweedFS, and single node deployments!
+> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
+
+Perfect for development, testing, learning SeaweedFS, and single-node deployments.

 ## Quick Start for S3 API on Docker ##

@@ -2,6 +2,7 @@ package command

 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/bits"
 	"net"
@@ -14,7 +15,10 @@ import (

 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -212,6 +216,9 @@ Example Usage:
 	weed mini                   # Use current directory
 	weed mini -dir=/data        # Custom data directory
 	weed mini -dir=/data -master.port=9444  # Custom master port
+	weed mini -dir=/data -bucket=my-bucket             # Pre-create an S3 bucket on startup
+	weed mini -dir=/data -bucket=bucket1,bucket2       # Pre-create multiple S3 buckets
+	weed mini -dir=/data -tableBucket=iceberg-tables   # Pre-create an S3 Tables bucket

 After starting, you can access:
 - Master UI:       http://localhost:9333
@@ -244,6 +251,8 @@ var (
 	miniS3Config                    = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")
 	miniIamConfig                   = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")
 	miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
+	miniBucket                      = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")
+	miniTableBucket                 = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.")
 )

 // getBindIp determines the bind IP address based on miniIp and miniBindIp flags
@@ -1020,6 +1029,22 @@ func runMini(cmd *Command, args []string) bool {
 		triggerMiniClientsShutdown(10 * time.Second)
 	})

+	// Create the requested bucket(s) (if any) before announcing readiness.
+	bucketSpec := *miniBucket
+	if bucketSpec == "" {
+		bucketSpec = os.Getenv("S3_BUCKET")
+	}
+	if err := ensureMiniBuckets(bucketSpec); err != nil {
+		glog.Warningf("failed to ensure buckets %q: %v", bucketSpec, err)
+	}
+	tableBucketSpec := *miniTableBucket
+	if tableBucketSpec == "" {
+		tableBucketSpec = os.Getenv("S3_TABLE_BUCKET")
+	}
+	if err := ensureMiniTableBuckets(tableBucketSpec); err != nil {
+		glog.Warningf("failed to ensure table buckets %q: %v", tableBucketSpec, err)
+	}
+
 	// Print welcome message after all services are running
 	printWelcomeMessage()

@@ -1484,3 +1509,111 @@ func printWelcomeMessage() {
 	fmt.Print(sb.String())
 	fmt.Println("")
 }
+
+// ensureMiniBuckets creates each named bucket on the embedded filer if it does
+// not already exist. bucketSpec is a comma-separated list (whitespace around
+// each name is trimmed); empty entries and an empty spec are no-ops so callers
+// who do not pass -bucket pay nothing. Per-bucket failures are logged and the
+// loop continues so a single bad name does not block creating the rest.
+func ensureMiniBuckets(bucketSpec string) error {
+	names := parseBucketList(bucketSpec)
+	if len(names) == 0 {
+		return nil
+	}
+
+	filerAddress := pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	const bucketsPath = "/buckets"
+	// Derive from miniClientsCtx so Ctrl+C cancels the bucket RPCs, and bound
+	// with a short timeout (per bucket) so a stalled filer cannot block the
+	// welcome message indefinitely.
+	return pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		for _, name := range names {
+			if err := s3bucket.VerifyS3BucketName(name); err != nil {
+				glog.Warningf("invalid bucket name %q: %v", name, err)
+				continue
+			}
+			ctx, cancel := context.WithTimeout(miniClientsCtx(), 5*time.Second)
+			_, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+				Directory: bucketsPath,
+				Name:      name,
+			})
+			if err == nil {
+				glog.V(0).Infof("bucket %s already exists", name)
+				cancel()
+				continue
+			}
+			if !errors.Is(err, filer_pb.ErrNotFound) {
+				glog.Warningf("lookup bucket %s: %v", name, err)
+				cancel()
+				continue
+			}
+			if err := filer_pb.DoMkdir(ctx, client, bucketsPath, name, nil); err != nil {
+				glog.Warningf("create bucket %s: %v", name, err)
+				cancel()
+				continue
+			}
+			cancel()
+			glog.V(0).Infof("created bucket %s", name)
+		}
+		return nil
+	})
+}
+
+// ensureMiniTableBuckets creates each named S3 Tables bucket on the embedded
+// filer if it does not already exist. bucketSpec is comma-separated; whitespace
+// is trimmed and duplicates are dropped. Per-bucket failures are logged so one
+// bad name does not block the rest. Buckets are owned by s3tables.DefaultAccountID
+// since mini does not yet model multi-account ownership.
+func ensureMiniTableBuckets(bucketSpec string) error {
+	names := parseBucketList(bucketSpec)
+	if len(names) == 0 {
+		return nil
+	}
+
+	filerAddress := pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	return pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		manager := s3tables.NewManager()
+		mgrClient := s3tables.NewManagerClient(client)
+		for _, name := range names {
+			ctx, cancel := context.WithTimeout(miniClientsCtx(), 5*time.Second)
+			req := &s3tables.CreateTableBucketRequest{Name: name}
+			var resp s3tables.CreateTableBucketResponse
+			err := manager.Execute(ctx, mgrClient, "CreateTableBucket", req, &resp, s3tables.DefaultAccountID)
+			cancel()
+			if err == nil {
+				glog.V(0).Infof("created table bucket %s", name)
+				continue
+			}
+			var s3Err *s3tables.S3TablesError
+			if errors.As(err, &s3Err) && s3Err.Type == s3tables.ErrCodeBucketAlreadyExists {
+				glog.V(0).Infof("table bucket %s already exists", name)
+				continue
+			}
+			glog.Warningf("create table bucket %s: %v", name, err)
+		}
+		return nil
+	})
+}
+
+// parseBucketList splits a comma-separated bucket spec into a deduplicated list
+// of trimmed, non-empty names, preserving the order they were given.
+func parseBucketList(spec string) []string {
+	if spec == "" {
+		return nil
+	}
+	seen := make(map[string]bool)
+	var names []string
+	for _, raw := range strings.Split(spec, ",") {
+		name := strings.TrimSpace(raw)
+		if name == "" || seen[name] {
+			continue
+		}
+		seen[name] = true
+		names = append(names, name)
+	}
+	return names
+}
@@ -0,0 +1,30 @@
+package command
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseBucketList(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want []string
+	}{
+		{"empty", "", nil},
+		{"single", "one", []string{"one"}},
+		{"multi", "one,two,three", []string{"one", "two", "three"}},
+		{"trims whitespace", " one , two , three ", []string{"one", "two", "three"}},
+		{"drops empty entries", "one,,two,", []string{"one", "two"}},
+		{"dedupes preserving order", "one,two,one,three,two", []string{"one", "two", "three"}},
+		{"only commas", ",,,", nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := parseBucketList(tt.in)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseBucketList(%q) = %v, want %v", tt.in, got, tt.want)
+			}
+		})
+	}
+}