diff --git a/backend/posix/posix.go b/backend/posix/posix.go index a0b14656..74e54240 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -126,6 +126,9 @@ const ( doFalloc = true skipFalloc = false + + // defaultConcurrency is the default limit for concurrent POSIX actions. + defaultConcurrency = 5000 ) // PosixOpts are the options for the Posix backend @@ -219,10 +222,18 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro newDirPerm: opts.NewDirPerm, forceNoTmpFile: opts.ForceNoTmpFile, validateBucketName: opts.ValidateBucketNames, - actionLimiter: semaphore.NewWeighted(int64(opts.Concurrency)), + actionLimiter: semaphore.NewWeighted(int64(concurrencyOrDefault(opts.Concurrency))), }, nil } +// concurrencyOrDefault returns n if it is positive, otherwise defaultConcurrency. +func concurrencyOrDefault(n int) int { + if n > 0 { + return n + } + return defaultConcurrency +} + func validateSubDir(root, dir string) (string, error) { absDir, err := filepath.Abs(dir) if err != nil { diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 90fa354c..23f9f7da 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -42,6 +42,9 @@ type ScoutfsOpts struct { // incorrect access to the filesystem. This is only needed if the // frontend is not already validating bucket names. ValidateBucketNames bool + // Concurrency sets the maximum number of concurrently running POSIX actions. + // Defaults to 5000 when unset or non-positive. + Concurrency int } var _ backend.Backend = &ScoutFS{} diff --git a/backend/scoutfs/scoutfs_compat.go b/backend/scoutfs/scoutfs_compat.go index 4b05b018..145290c1 100644 --- a/backend/scoutfs/scoutfs_compat.go +++ b/backend/scoutfs/scoutfs_compat.go @@ -81,6 +81,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) { NewDirPerm: opts.NewDirPerm, VersioningDir: opts.VersioningDir, ValidateBucketNames: opts.ValidateBucketNames, + Concurrency: opts.Concurrency, }) if err != nil { return nil, err diff --git a/cmd/versitygw/scoutfs.go b/cmd/versitygw/scoutfs.go index 8ea4a77d..3af73e44 100644 --- a/cmd/versitygw/scoutfs.go +++ b/cmd/versitygw/scoutfs.go @@ -99,6 +99,13 @@ move interfaces as well as support for tiered filesystems.`, EnvVars: []string{"VGW_DISABLE_NOARCHIVE"}, Destination: &disableNoArchive, }, + &cli.IntFlag{ + Name: "concurrency", + Usage: "maximum concurrent actions allowed", + EnvVars: []string{"VGW_POSIX_CONCURRENCY"}, + Value: 5000, + Destination: &actionsConcurrency, + }, }, } } @@ -112,6 +119,10 @@ func runScoutfs(ctx *cli.Context) error { return fmt.Errorf("invalid directory permissions: %d", dirPerms) } + if actionsConcurrency <= 0 { + return fmt.Errorf("concurrency must be positive, got %d", actionsConcurrency) + } + var opts scoutfs.ScoutfsOpts opts.GlacierMode = glacier opts.ChownUID = chownuid @@ -122,6 +133,7 @@ func runScoutfs(ctx *cli.Context) error { opts.VersioningDir = versioningDir opts.ValidateBucketNames = disableStrictBucketNames opts.SetProjectID = setProjectID + opts.Concurrency = actionsConcurrency be, err := scoutfs.New(ctx.Args().Get(0), opts) if err != nil {