feat: add option to disable use of O_TMPFILE

O_TMPFILE can fail if the location we need to link the final
file is not within the same filesystem. This can happen if
there are different filesystem mounts within a bucket or if
using zfs nested datasets within a bucket.

Fixes #1194
Fixes #1035

cmd/versitygw/main.go

@@ -524,19 +524,19 @@ func initFlags() []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:        "ipa-user",
-			Usage:       "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
+			Usage:       "Username used to connect to FreeIPA (requires permissions to read user vault contents)",
 			EnvVars:     []string{"VGW_IPA_USER"},
 			Destination: &ipaUser,
 		},
 		&cli.StringFlag{
 			Name:        "ipa-password",
-			Usage:       "Password of the user used to connect to FreeIPA.",
+			Usage:       "Password of the user used to connect to FreeIPA",
 			EnvVars:     []string{"VGW_IPA_PASSWORD"},
 			Destination: &ipaPassword,
 		},
 		&cli.BoolFlag{
 			Name:        "ipa-insecure",
-			Usage:       "Verify TLS certificate of FreeIPA server. Default is 'true'.",
+			Usage:       "Disable verify TLS certificate of FreeIPA server",
 			EnvVars:     []string{"VGW_IPA_INSECURE"},
 			Destination: &ipaInsecure,
 		},

cmd/versitygw/posix.go

@@ -31,6 +31,7 @@ var (
 	dirPerms           uint
 	sidecar            string
 	nometa             bool
+	forceNoTmpFile     bool
 )
 
 func posixCommand() *cli.Command {
@@ -93,6 +94,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_META_NONE"},
 				Destination: &nometa,
 			},
+			&cli.BoolFlag{
+				Name:        "disableotmp",
+				Usage:       "disable O_TMPFILE support for new objects",
+				EnvVars:     []string{"VGW_DISABLE_OTMP"},
+				Destination: &forceNoTmpFile,
+			},
 		},
 	}
 }
@@ -113,11 +120,12 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	opts := posix.PosixOpts{
-		ChownUID:      chownuid,
-		ChownGID:      chowngid,
-		BucketLinks:   bucketlinks,
-		VersioningDir: versioningDir,
-		NewDirPerm:    fs.FileMode(dirPerms),
+		ChownUID:       chownuid,
+		ChownGID:       chowngid,
+		BucketLinks:    bucketlinks,
+		VersioningDir:  versioningDir,
+		NewDirPerm:     fs.FileMode(dirPerms),
+		ForceNoTmpFile: forceNoTmpFile,
 	}
 
 	var ms meta.MetadataStorer