################################### # VersityGW systemd configuration # ################################### # Copy this file to /etc/versitygw.d/ and rename it to a unique service name. # For example, if the service name is "mygateway", then the file should be # named /etc/versitygw.d/mygateway.conf. # The systemd template file /lib/systemd/system/versitygw@.service will # automatically load the configuration file for the service name. # To start the gateway, use the following command: # systemctl start versitygw@mygateway # To enable the gateway to start on boot, use the following command: # systemctl enable versitygw@mygateway # To stop the gateway, use the following command: # systemctl stop versitygw@mygateway # There can be multiple gateway services running on the same host. Each # gateway service must have a unique service name with a unique configuration # file in /etc/versitygw.d/. They must also listen on different ports and/or # interfaces using the VGW_PORT option. ############################## # VersityGW Required Options # ############################## # VGW_BACKEND must be defined, and must be one of: posix, scoutfs, s3, azure, # or plugin # This defines the backend that the VGW will use for data access. VGW_BACKEND=posix # When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined # as the the top level directory for the gateway. # All sub directories of the top level directory are treated as buckets, # and all files/directories below the "bucket directory" are treated as # the objects. The object name is split on "/" separator to translate # to posix storage. # For example: # (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot # bucket: mybucket # object: a/b/c/myobject # will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject VGW_BACKEND_ARG= ############################ # VersityGW Global Options # ############################ # commented options are the default values # The following must be set, and do not have default values # The access and secret options will specify the root account credentials. # The root account is granted full authorization to all API requests after # authentication. ROOT_ACCESS_KEY_ID= ROOT_SECRET_ACCESS_KEY= # The following are optional, and have the default values as listed # The VGW_PORT option will specify the listening port for the S3 server. # This option can use either the form : which will listen only # on the network interface that matches the IP on the specified port, or # : which will listen on all network interfaces on the specified port. # The spec can either be IP dotted notation or a resolvable hostname. # The spec can either be a numeric port or the service name typically # in /etc/services. #VGW_PORT=:7070 # The VGW_REGION option will specify the region that the S3 server will # report to clients. This option is optional, and defaults to "us-east-1". #VGW_REGION=us-east-1 # The VGW_CERT and VGW_KEY options will specify the SSL certificate and # private key that the S3 server will use for SSL connections. This option # is optional, and defaults to not using SSL. #VGW_CERT= #VGW_KEY= # The VGW_ADMIN_PORT option will specify the listening port for the admin # server. The admin server endpoint can optionally be set to listen on a # different interface or port than the S3 service. This allows for better # control of firewall restrictions to the admin endpoint. The certs for this # can be different certs than specified for the S3 service. The default when # these are not specified is to have the admin server listen on the same # endpoint as the S3 service. # When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin # server will use SSL. #VGW_ADMIN_PORT= #VGW_ADMIN_CERT= #VGW_ADMIN_CERT_KEY= # The VGW_QUIET option when set will supress the S3 server request summary # logging to stdout. #VGW_QUIET=false # The VGW_HEALTH option when set will specify the URL to accept health checks # on. The health check endpoint is often used for load balancers to verify # gateway is alive. The health endpoint masks any bucket with this setting. # For example, if the health endpoint is set to /health, the gateway will not # allow creating or listing contents of a bucket called "health". The health # endpoint is unauthenticated, and returns a 200 status for GET. #VGW_HEALTH= # Enable VGW_READ_ONLY to only allow read operations to the S3 server. No write # operations will be allowed. #VGW_READ_ONLY=false # The VGW_VIRTUAL_DOMAIN option enables the virtual host style bucket # addressing. The path style addressing is the default, and remains enabled # even when virtual host style is enabled. The VGW_VIRTUAL_DOMAIN option # specifies the domain name that will be used for the virtual host style # addressing. For virtual addressing, access to a bucket is in the request # form: # https://./ # for example: https://mybucket.example.com/ where # VGW_VIRTUAL_DOMAIN=example.com # and all subdomains of VGW_VIRTUAL_DOMAIN should be reserved for buckets. # This means that virtual host addressing will generally require a DNS # entry for each bucket that needs to be accessed. # The default path style request is of the form: # https:/// #VGW_VIRTUAL_DOMAIN= # By default, versitygw will enforce similar bucket naming rules as described # in https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html # Set to true to allow legacy or non-DNS-compliant bucket names by skipping # strict validation checks. #VGW_DISABLE_STRICT_BUCKET_NAMES=false ############### # Access Logs # ############### # The VGW_ACCESS_LOG option when set will specify the file to log all S3 # server requests to. This option is optional, and defaults to not logging. # It is suggested to use absolute paths for the server log file because the # server may chdir into the backend root directory and change locations for # relative paths. # The log file format follows the AWS S3 access log format documented in # https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html. #VGW_ACCESS_LOG= # The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the # S3 server request access logs to. The access logs are JSON encoded when # sent to the webhook. #VGW_LOG_WEBHOOK_URL= ############## # Event Logs # ############## # The gateway events are similar to AWS S3 events, and are documented in the # wiki: # https://github.com/versity/versitygw/wiki/Events-Notifications. # The VGW_EVENT_FILTER option specifies a config file that contains the # event filter rules. The event filter rules are used to determine which # events are sent to the configured event services. # Use the following to generate a default rules file in /etc/versitygw.d/: # versitygw utils gen-event-filter-config -p /etc/versitygw.d # The resulting file, /etc/versitygw.d/event_config.json, can be modified and # specified in the VGW_EVENT_FILTER option. # When VGW_EVENT_FILTER is not specified, all events are sent to the configured # event service. #VGW_EVENT_FILTER= # Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL, # VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all # configured bucket events will be sent to the kafka service. #VGW_EVENT_KAFKA_URL= #VGW_EVENT_KAFKA_TOPIC= #VGW_EVENT_KAFKA_KEY= # Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL # and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be # sent to the the NATS messaging service. #VGW_EVENT_NATS_URL= #VGW_EVENT_NATS_TOPIC= # Bucket events can be sent to a RabbitMQ messaging service. When # VGW_EVENT_RABBITMQ_URL is specified, events will be published to the specified # exchange (VGW_EVENT_RABBITMQ_EXCHANGE) using the routing key # (VGW_EVENT_RABBITMQ_ROUTING_KEY). If exchange is blank the default exchange is # used. If routing key is blank, it will be left empty (the server can bind a # queue with an empty binding key or you can set an explicit key). # Example URL formats: # amqp://user:pass@rabbitmq:5672/ # amqps://user:pass@rabbitmq:5671/vhost #VGW_EVENT_RABBITMQ_URL= #VGW_EVENT_RABBITMQ_EXCHANGE= #VGW_EVENT_RABBITMQ_ROUTING_KEY= # Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is # specified, all configured bucket events will be sent to the webhook. #VGW_EVENT_WEBHOOK_URL= # Bucket events can be filtered for any of the above event types. The # VGW_EVENT_FILTER option specifies a config file that contains the event # filter rules. The event filter rules are used to determine which events are # sent to the configured event services. Run: # versitygw utils gen-event-filter-config --path . # to generate a default rules file "event_config.json" in the current directory. #VGW_EVENT_FILTER= ####################### # Debug / Diagnostics # ####################### # The VGW_DEBUG option enables verbose debug log output to stdout. This output # includes details for signature verification steps. This is generally only # useful for debugging the S3 server, and should not be used in production. #VGW_DEBUG=false # The VGW_PPROF option enables the pprof HTTP server for profiling the S3 # server. See the following for more information: # https://pkg.go.dev/net/http/pprof # To enable, set the VGW_PPROF option to the listening address for the pprof # server. For example, to listen on localhost port 6060, set the option to # "localhost:6060". #VGW_PPROF= ################ # IAM services # ################ # The VGW_IAM_DIR option will enable the internal IAM service with accounts # stored in a file under the specified directory. This is provided to minimize # dependencies on outside services for basic functionality. The local account # files are plain text and only protected with file permissions. This IAM # service is added for convenience, but is not considered as secure or scalable # as a dedicated IAM service. #VGW_IAM_DIR= # The Vault options will enable the Vault IAM service with accounts stored in # the HashiCorp Vault service. The Vault URL is the address and port of the # Vault server with the format :. A root taken can be used for # testing, but it is recommended to use the role based authentication in # production. The Vault server certificate, client certificate, and client # certificate key are optional, and will default to not verifying the server # certificate and not using client certificates. The Vault server certificate # is used to verify the Vault server, and the client certificate and key are # used to authenticate the gateway to the Vault server. See wiki documentation # for an example of using Vault in dev mode with the gateway. #VGW_IAM_VAULT_ENDPOINT_URL= #VGW_IAM_VAULT_SECRET_STORAGE_PATH= #VGW_IAM_VAULT_MOUNT_PATH= #VGW_IAM_VAULT_ROOT_TOKEN= #VGW_IAM_VAULT_ROLE_ID= #VGW_IAM_VAULT_ROLE_SECRET= #VGW_IAM_VAULT_SERVER_CERT= #VGW_IAM_VAULT_CLIENT_CERT= #VGW_IAM_VAULT_CLIENT_CERT_KEY= # The VGW_S3 IAM service is similar to the internal IAM service, but instead # stores the account information JSON encoded in an S3 object. This should use # a bucket that is not accessible to general users when using s3 backend to # prevent access to account credentials. This IAM service is added for # convenience, but is not considered as secure or scalable as a dedicated IAM # service. #VGW_S3_IAM_ACCESS_KEY= #VGW_S3_IAM_SECRET_KEY= #VGW_S3_IAM_REGION= #VGW_S3_IAM_ENDPOINT= #VGW_S3_IAM_BUCKET= #VGW_S3_IAM_NO_VERIFY= # The LDAP options will enable the LDAP IAM service with accounts stored in an # external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR, # and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access, # secret credentials and role respectively. The other options are used to # connect to the LDAP service. #VGW_IAM_LDAP_URL= #VGW_IAM_LDAP_BASE_DN= #VGW_IAM_LDAP_BIND_DN= #VGW_IAM_LDAP_BIND_PASS= #VGW_IAM_LDAP_QUERY_BASE= #VGW_IAM_LDAP_OBJECT_CLASSES= #VGW_IAM_LDAP_ACCESS_ATR= #VGW_IAM_LDAP_SECRET_ATR= #VGW_IAM_LDAP_ROLE_ATR= #VGW_IAM_LDAP_USER_ID_ATR= #VGW_IAM_LDAP_GROUP_ID_ATR= # Disable TLS certificate verification for LDAP connections (insecure, allows # self-signed certificates). This should only be used in testing environments # or when using self-signed certificates. The default is false (verification # enabled). #VGW_IAM_LDAP_TLS_SKIP_VERIFY=false # The FreeIPA options will enable the FreeIPA IAM service with accounts stored # in an external FreeIPA service. Currently the FreeIPA IAM service only # supports account retrieval. Creating and modifying accounts must be done # outside of the versitygw service. # FreeIPA server url e.g. https://ipa.example.test #VGW_IPA_HOST= # A name of the user vault containing their secret #VGW_IPA_VAULT_NAME= # Username used to connect to FreeIPA (requires permissions to read user vault # contents) #VGW_IPA_USER= # Password of the user used to connect to FreeIPA #VGW_IPA_PASSWORD= # Disable verify TLS certificate of FreeIPA server #VGW_IPA_INSECURE=false # FreeIPA IAM debug output #VGW_IPA_DEBUG=false ############### # IAM caching # ############### # The IAM cache is intended to ease the load on the IAM service and increase # the Gateway performance by caching accounts and credentials for the TTL time # interval. Disabling this will cause a request to the configured IAM service # for each incoming request to retrieve the corresponding account credentials. # The cache is enabled by default. The TTL specifies how long to cache # credentials, and the prune value determines the interval for expired entries # to be removed from the cache. Increasing the TTL may lessen the load on the # IAM service backend, but may have out of date account info until the next # interval. Increasing the prune value may reduce memory use at the cost of # added CPU to check cache expirations. #VGW_IAM_CACHE_DISABLE=false #VGW_IAM_CACHE_TTL=120 #VGW_IAM_CACHE_PRUNE=3600 ########### # Metrics # ########### # The metrics service name is a tag that is added to all metrics to help # identify the source of the metrics. This is especially useful when multiple # gateways are running. The default is the hostname of the system. #VGW_METRICS_SERVICE_NAME=$HOSTNAME # The metrics service will send metrics to the configured statsd servers. The # servers are specified as a comma separated list of host:port pairs. The # default is to not send metrics to any statsd servers. The gateway uses # InfluxDB flavor of statsd metrics tags for the StatsD metrics type. #VGW_METRICS_STATSD_SERVERS= # The metrics service will send metrics to the configured dogstatsd servers. # The servers are specified as a comma separated list of host:port pairs. The # default is to not send metrics to any dogstatsd servers. Generally # DataDog recommends installing a local agent to collect metrics and forward # them to the DataDog service. In this case the option value would be the # local agent address: 127.0.0.1:8125. #VGW_METRICS_DOGSTATS_SERVERS= ###################################### # VersityGW Backend Specific Options # ###################################### ######### # posix # ######### # The posix backend translates S3 requests to file access in a local filesystem. # The posix backend requires a filesystem that supports extended attributes. # The top level directory for the gateway must be provided. All sub directories # of the top level directory are treated as buckets, and all files/directories # below the "bucket directory" are treated as the objects. The object # name is split on "/" separator to translate to posix storage. # For example: # top level (VGW_BACKEND_ARG): /mnt/fs/gwroot # bucket: mybucket # object: a/b/c/myobject # will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject # The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to # change the ownership of newly created files and directories to the IAM # account UID/GID. #VGW_CHOWN_UID=false #VGW_CHOWN_GID=false # The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links # to directories at the top level gateway directory as buckets. #VGW_BUCKET_LINKS=false # The default permissions mode when creating new directories is 0755. Use # VGW_DIR_PERMS option to set a different mode for any new directory that the # gateway creates. This applies to buckets created through the gateway as well # as any parent directories automatically created with object uploads. #VGW_DIR_PERMS=0755 # To enable object versions, the VGW_VERSIONING_DIR option must be set to the # directory that will be used to store the object versions. The version # directory must NOT be a subdirectory of the VGW_BACKEND_ARG directory. #VGW_VERSIONING_DIR= # The gateway uses xattrs to store metadata for objects by default. For systems # that do not support xattrs, the VGW_META_SIDECAR option can be set to a # directory that will be used to store the metadata for objects. This is # currently experimental, and may have issues for some edge cases. #VGW_META_SIDECAR= # The VGW_META_NONE option will disable the metadata functionality for the # gateway. This will cause the gateway to not store any metadata for objects # or buckets. This include bucket ACLs and Policy. This may be useful for # read only access to pre-existing data where the gateway should not modify # the data. It is recommened to enable VGW_READ_ONLY (Global Options) along # with this. #VGW_META_NONE=false # The gateway will use O_TMPFILE for writing objects while uploading and # link the file to the final object name when the upload is complete if the # filesystem supports O_TMPFILE. This creates an atomic object creation # that is not visible to other clients or racing uploads until the upload # is complete. This will not work if there is a different filesystem mounted # below the bucket level than where the bucket resides. The VGW_DISABLE_OTMP # option can be set to true to disable this functionality and force the fallback # mode when O_TMPFILE is not available. This fallback will create a temporary # file in the bucket directory and rename it to the final object name when # the upload is complete if the final location is in the same filesystem, or # copy the file to the final location if the final location is in a different # filesystem. This fallback mode is still atomic, but may be less efficient # than O_TMPFILE when the data needs to be copied into the final location. #VGW_DISABLE_OTMP=false ########### # scoutfs # ########### # The scoutfs backend requires a ScoutFS filesystem type for the backend # path. The object to posix name mappings follow the same rules as posix for # scoutfs. The glacier mode functionality requires ScoutAM to be configured # for tiering data from the ScoutFS filesystem to a mass stroage system. # The mass storage system is often one or more tape libraries. Due to the # high latency of tape, the glacier mode functionality is designed to # give feedback to clients about object state and offer ability to request # data to be staged back to disk without the client dealing with long # request timeout settings. # The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior. # GET object: if file offline, return invalid object state # HEAD object: if file offline, set obj storage class to GLACIER # if file offline and staging, x-amz-restore: ongoing-request="true" # if file offline and not staging, x-amz-restore: ongoing-request="false" # if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT" # note: this expiry-date is not used but provided for client glacier compatibility # ListObjects: if file offline, set obj storage class to GLACIER # RestoreObject: add batch stage request to file #VGW_SCOUTFS_GLACIER=false # The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to # change the ownership of newly created files and directories to the IAM # account UID/GID. #VGW_CHOWN_UID=false #VGW_CHOWN_GID=false # The VGW_SET_PROJECT_ID option will enable setting account defined ProjectID # for newly created buckets, files, and directories if the account ProjectID # is greater than 0 and the filesystem format version supports project IDs. #VGW_SET_PROJECT_ID=false # The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links # to directories at the top level gateway directory as buckets. #VGW_BUCKET_LINKS=false # The default permissions mode when creating new directories is 0755. Use # VGW_DIR_PERMS option to set a different mode for any new directory that the # gateway creates. This applies to buckets created through the gateway as well # as any parent directories automatically created with object uploads. #VGW_DIR_PERMS=0755 # To enable object versions, the VGW_VERSIONING_DIR option must be set to the # directory that will be used to store the object versions. The version # directory must NOT be a subdirectory of the VGW_BACKEND_ARG directory. # There may be implications for archive policy updates to include version # directory as well. It is recommended to discuss archive implications of # versioning with Versity support before enabling on an archiving filesystem. #VGW_VERSIONING_DIR= # The default behavior of the gateway is to automatically set the noarchive # flag on the multipart upload parts while the multipart upload is in progress. # This is to prevent the parts from being archived since they are temporary # and will be deleted after the multipart upload is completed or aborted. The # VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior. #VGW_DISABLE_NOARCHIVE=false ###### # s3 # ###### # The s3 backend allows the gateway to forward requests to an S3 compatible # service. This allows the gateway to act as a proxy for another S3 service. # The backend S3 access is all done with a single configured account. The # gateway will manage incoming multi-tenant access with the gateway configured # IAM service. This gives stroage admins the ability to manage local gateway # accounts while maintaining full control and a single account for the backend # S3 service. # When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must # be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will # default to "us-east-1" and "https://s3.amazonaws.com" respectively. #VGW_S3_ACCESS_KEY= #VGW_S3_SECRET_KEY= #VGW_S3_REGION= #VGW_S3_ENDPOINT= #VGW_S3_DISABLE_CHECKSUM=false #VGW_S3_SSL_SKIP_VERIFY=false #VGW_S3_DEBUG=false ######## # azure # ######## # The azure backend allows the gateway to store objects in Azure Blob Storage. # Buckets created through the gateway map to blob containers within the # configured storage account. This backend is useful when existing workflows # expect an S3-compatible interface while data resides in Azure. # When the azure backend is selected, configure credentials with one of the # following approaches: # - Shared key: Define AZ_ACCOUNT_NAME with the storage account name and # AZ_ACCESS_KEY with the corresponding account key. # - SAS token: Set AZ_SAS_TOKEN to an account or container scoped SAS token. # Provide AZ_ENDPOINT if the token does not implicitly define the endpoint. # - Default Azure credentials: Leave AZ_ACCOUNT_NAME and AZ_ACCESS_KEY blank # and configure the standard Azure identity environment variables supported # by the DefaultAzureCredential chain (e.g. AZURE_CLIENT_ID, AZURE_TENANT_ID, # AZURE_CLIENT_SECRET, managed identity, etc.). # Use AZ_ENDPOINT to override the service URL (for example when targeting # Azurite or a sovereign cloud). If unset, it defaults to # https://.blob.core.windows.net/ when an account name is provided. #AZ_ACCOUNT_NAME= #AZ_ACCESS_KEY= #AZ_SAS_TOKEN= #AZ_ENDPOINT= ########## # plugin # ########## # The plugin backend loads a Go plugin shared object that exposes a variable # named "Backend" of type *plugins.BackendPlugin. The gateway uses the # exported constructor to create the backend implementation at runtime. # Set VGW_BACKEND_ARG to the absolute path of the compiled plugin (.so) file. # The path must be readable by the gateway service account and remain stable # across restarts. #VGW_BACKEND_ARG=/usr/lib/versitygw/plugins/example.so # Provide the plugin-specific configuration file path via VGW_PLUGIN_CONFIG. # The gateway automatically forwards this value to the plugin backend when it # starts up. #VGW_PLUGIN_CONFIG=/etc/versitygw.d/example-plugin.conf