versitygw/extra/example.conf

###################################
# VersityGW systemd configuration #
###################################

# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
# For example, if the service name is "mygateway", then the file should be
# named /etc/versitygw.d/mygateway.conf.
# The systemd template file /lib/systemd/system/versitygw@.service will
# automatically load the configuration file for the service name.
# To start the gateway, use the following command:
# systemctl start versitygw@mygateway
# To enable the gateway to start on boot, use the following command:
# systemctl enable versitygw@mygateway
# To stop the gateway, use the following command:
# systemctl stop versitygw@mygateway

# There can be multiple gateway services running on the same host. Each
# gateway service must have a unique service name with a unique configuration
# file in /etc/versitygw.d/. They must also listen on different ports and/or
# interfaces using the VGW_PORT option.

##############################
# VersityGW Required Options #
##############################

# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, s3, azure,
# or plugin
# This defines the backend that the VGW will use for data access.
VGW_BACKEND=posix

# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
# as the the top level directory for the gateway.
# All sub directories of the top level directory are treated as buckets,
# and all files/directories below the "bucket directory" are treated as
# the objects. The object name is split on "/" separator to translate
# to posix storage.
# For example:
# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
# bucket: mybucket
# object: a/b/c/myobject
# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
VGW_BACKEND_ARG=

############################
# VersityGW Global Options #
############################

# commented options are the default values

# The following must be set, and do not have default values
# The access and secret options will specify the root account credentials.
# The root account is granted full authorization to all API requests after
# authentication.
ROOT_ACCESS_KEY_ID=
ROOT_SECRET_ACCESS_KEY=

# The following are optional, and have the default values as listed

# The VGW_PORT option will specify the listening port for the S3 server.
# This option can use either the form <ip>:<port> which will listen only
# on the network interface that matches the IP on the specified port, or
# :<port> which will listen on all network interfaces on the specified port.
# The <ip> spec can either be IP dotted notation or a resolvable hostname.
# The <port> spec can either be a numeric port or the service name typically
# in /etc/services.
#VGW_PORT=:7070

# The VGW_REGION option will specify the region that the S3 server will
# report to clients. This option is optional, and defaults to "us-east-1".
#VGW_REGION=us-east-1

# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
# private key that the S3 server will use for SSL connections. This option
# is optional, and defaults to not using SSL.
#VGW_CERT=
#VGW_KEY=

# The VGW_ADMIN_PORT option will specify the listening port for the admin
# server. The admin server endpoint can optionally be set to listen on a
# different interface or port than the S3 service. This allows for better
# control of firewall restrictions to the admin endpoint. The certs for this
# can be different certs than specified for the S3 service. The default when
# these are not specified is to have the admin server listen on the same
# endpoint as the S3 service.
# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
# server will use SSL.
#VGW_ADMIN_PORT=
#VGW_ADMIN_CERT=
#VGW_ADMIN_CERT_KEY=

# The VGW_QUIET option when set will supress the S3 server request summary
# logging to stdout.
#VGW_QUIET=false

# The VGW_HEALTH option when set will specify the URL to accept health checks
# on. The health check endpoint is often used for load balancers to verify
# gateway is alive. The health endpoint masks any bucket with this setting.
# For example, if the health endpoint is set to /health, the gateway will not
# allow creating or listing contents of a bucket called "health". The health
# endpoint is unauthenticated, and returns a 200 status for GET.
#VGW_HEALTH=

# Enable VGW_READ_ONLY to only allow read operations to the S3 server. No write
# operations will be allowed.
#VGW_READ_ONLY=false

# The VGW_VIRTUAL_DOMAIN option enables the virtual host style bucket
# addressing. The path style addressing is the default, and remains enabled
# even when virtual host style is enabled. The VGW_VIRTUAL_DOMAIN option
# specifies the domain name that will be used for the virtual host style
# addressing. For virtual addressing, access to a bucket is in the request
# form:
# https://<bucket>.<VGW_VIRTUAL_DOMAIN>/
# for example: https://mybucket.example.com/ where
# VGW_VIRTUAL_DOMAIN=example.com
# and all subdomains of VGW_VIRTUAL_DOMAIN should be reserved for buckets.
# This means that virtual host addressing will generally require a DNS
# entry for each bucket that needs to be accessed.
# The default path style request is of the form:
# https://<VGW_ENDPOINT>/<bucket>
#VGW_VIRTUAL_DOMAIN=

# By default, versitygw will enforce similar bucket naming rules as described
# in https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
# Set to true to allow legacy or non-DNS-compliant bucket names by skipping
# strict validation checks.
#VGW_DISABLE_STRICT_BUCKET_NAMES=false

###############
# Access Logs #
###############

# The VGW_ACCESS_LOG option when set will specify the file to log all S3
# server requests to. This option is optional, and defaults to not logging.
# It is suggested to use absolute paths for the server log file because the
# server may chdir into the backend root directory and change locations for
# relative paths.
# The log file format follows the AWS S3 access log format documented in
# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
#VGW_ACCESS_LOG=

# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
# S3 server request access logs to. The access logs are JSON encoded when
# sent to the webhook.
#VGW_LOG_WEBHOOK_URL=

##############
# Event Logs #
##############

# The gateway events are similar to AWS S3 events, and are documented in the
# wiki:
# https://github.com/versity/versitygw/wiki/Events-Notifications.

# The VGW_EVENT_FILTER option specifies a config file that contains the
# event filter rules. The event filter rules are used to determine which
# events are sent to the configured event services.
# Use the following to generate a default rules file in /etc/versitygw.d/:
# versitygw utils gen-event-filter-config -p /etc/versitygw.d
# The resulting file, /etc/versitygw.d/event_config.json, can be modified and
# specified in the VGW_EVENT_FILTER option.
# When VGW_EVENT_FILTER is not specified, all events are sent to the configured
# event service.
#VGW_EVENT_FILTER=

# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
# configured bucket events will be sent to the kafka service.
#VGW_EVENT_KAFKA_URL=
#VGW_EVENT_KAFKA_TOPIC=
#VGW_EVENT_KAFKA_KEY=

# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
# and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be
# sent to the the NATS messaging service.
#VGW_EVENT_NATS_URL=
#VGW_EVENT_NATS_TOPIC=

# Bucket events can be sent to a RabbitMQ messaging service. When
# VGW_EVENT_RABBITMQ_URL is specified, events will be published to the specified
# exchange (VGW_EVENT_RABBITMQ_EXCHANGE) using the routing key
# (VGW_EVENT_RABBITMQ_ROUTING_KEY). If exchange is blank the default exchange is
# used. If routing key is blank, it will be left empty (the server can bind a
# queue with an empty binding key or you can set an explicit key).
# Example URL formats:
#   amqp://user:pass@rabbitmq:5672/
#   amqps://user:pass@rabbitmq:5671/vhost
#VGW_EVENT_RABBITMQ_URL=
#VGW_EVENT_RABBITMQ_EXCHANGE=
#VGW_EVENT_RABBITMQ_ROUTING_KEY=

# Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is
# specified, all configured bucket events will be sent to the webhook.
#VGW_EVENT_WEBHOOK_URL=

# Bucket events can be filtered for any of the above event types. The
# VGW_EVENT_FILTER option specifies a config file that contains the event
# filter rules. The event filter rules are used to determine which events are
# sent to the configured event services. Run:
# versitygw utils gen-event-filter-config --path .
# to generate a default rules file "event_config.json" in the current directory.
#VGW_EVENT_FILTER=

#######################
# Debug / Diagnostics #
#######################

# The VGW_DEBUG option enables verbose debug log output to stdout. This output
# includes details for signature verification steps. This is generally only
# useful for debugging the S3 server, and should not be used in production.
#VGW_DEBUG=false

# The VGW_PPROF option enables the pprof HTTP server for profiling the S3
# server. See the following for more information:
# https://pkg.go.dev/net/http/pprof
# To enable, set the VGW_PPROF option to the listening address for the pprof
# server. For example, to listen on localhost port 6060, set the option to
# "localhost:6060".
#VGW_PPROF=

################
# IAM services #
################

# The VGW_IAM_DIR option will enable the internal IAM service with accounts
# stored in a file under the specified directory. This is provided to minimize
# dependencies on outside services for basic functionality. The local account
# files are plain text and only protected with file permissions. This IAM
# service is added for convenience, but is not considered as secure or scalable
# as a dedicated IAM service.
#VGW_IAM_DIR=

# The Vault options will enable the Vault IAM service with accounts stored in
# the HashiCorp Vault service. The Vault URL is the address and port of the
# Vault server with the format <IP/host>:<port>. A root taken can be used for
# testing, but it is recommended to use the role based authentication in
# production. The Vault server certificate, client certificate, and client
# certificate key are optional, and will default to not verifying the server
# certificate and not using client certificates. The Vault server certificate
# is used to verify the Vault server, and the client certificate and key are
# used to authenticate the gateway to the Vault server. See wiki documentation
# for an example of using Vault in dev mode with the gateway.
#VGW_IAM_VAULT_ENDPOINT_URL=
#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
#VGW_IAM_VAULT_MOUNT_PATH=
#VGW_IAM_VAULT_ROOT_TOKEN=
#VGW_IAM_VAULT_ROLE_ID=
#VGW_IAM_VAULT_ROLE_SECRET=
#VGW_IAM_VAULT_SERVER_CERT=
#VGW_IAM_VAULT_CLIENT_CERT=
#VGW_IAM_VAULT_CLIENT_CERT_KEY=

# The VGW_S3 IAM service is similar to the internal IAM service, but instead
# stores the account information JSON encoded in an S3 object. This should use
# a bucket that is not accessible to general users when using s3 backend to
# prevent access to account credentials. This IAM service is added for
# convenience, but is not considered as secure or scalable as a dedicated IAM
# service.
#VGW_S3_IAM_ACCESS_KEY=
#VGW_S3_IAM_SECRET_KEY=
#VGW_S3_IAM_REGION=
#VGW_S3_IAM_ENDPOINT=
#VGW_S3_IAM_BUCKET=
#VGW_S3_IAM_NO_VERIFY=

# The LDAP options will enable the LDAP IAM service with accounts stored in an
# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
# secret credentials and role respectively. The other options are used to
# connect to the LDAP service.
#VGW_IAM_LDAP_URL=
#VGW_IAM_LDAP_BASE_DN=
#VGW_IAM_LDAP_BIND_DN=
#VGW_IAM_LDAP_BIND_PASS=
#VGW_IAM_LDAP_QUERY_BASE=
#VGW_IAM_LDAP_OBJECT_CLASSES=
#VGW_IAM_LDAP_ACCESS_ATR=
#VGW_IAM_LDAP_SECRET_ATR=
#VGW_IAM_LDAP_ROLE_ATR=
#VGW_IAM_LDAP_USER_ID_ATR=
#VGW_IAM_LDAP_GROUP_ID_ATR=
# Disable TLS certificate verification for LDAP connections (insecure, allows
# self-signed certificates). This should only be used in testing environments
# or when using self-signed certificates. The default is false (verification
# enabled).
#VGW_IAM_LDAP_TLS_SKIP_VERIFY=false

# The FreeIPA options will enable the FreeIPA IAM service with accounts stored
# in an external FreeIPA service. Currently the FreeIPA IAM service only
# supports account retrieval. Creating and modifying accounts must be done
# outside of the versitygw service.
# FreeIPA server url e.g. https://ipa.example.test
#VGW_IPA_HOST=
# A name of the user vault containing their secret
#VGW_IPA_VAULT_NAME=
# Username used to connect to FreeIPA (requires permissions to read user vault
# contents)
#VGW_IPA_USER=
# Password of the user used to connect to FreeIPA
#VGW_IPA_PASSWORD=
# Disable verify TLS certificate of FreeIPA server
#VGW_IPA_INSECURE=false
# FreeIPA IAM debug output
#VGW_IPA_DEBUG=false

###############
# IAM caching #
###############

# The IAM cache is intended to ease the load on the IAM service and increase
# the Gateway performance by caching accounts and credentials for the TTL time
# interval. Disabling this will cause a request to the configured IAM service
# for each incoming request to retrieve the corresponding account credentials.
# The cache is enabled by default. The TTL specifies how long to cache
# credentials, and the prune value determines the interval for expired entries
# to be removed from the cache. Increasing the TTL may lessen the load on the
# IAM service backend, but may have out of date account info until the next
# interval. Increasing the prune value may reduce memory use at the cost of
# added CPU to check cache expirations.
#VGW_IAM_CACHE_DISABLE=false
#VGW_IAM_CACHE_TTL=120
#VGW_IAM_CACHE_PRUNE=3600

###########
# Metrics #
###########

# The metrics service name is a tag that is added to all metrics to help
# identify the source of the metrics. This is especially useful when multiple
# gateways are running. The default is the hostname of the system.
#VGW_METRICS_SERVICE_NAME=$HOSTNAME

# The metrics service will send metrics to the configured statsd servers. The
# servers are specified as a comma separated list of host:port pairs. The
# default is to not send metrics to any statsd servers. The gateway uses
# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
#VGW_METRICS_STATSD_SERVERS=

# The metrics service will send metrics to the configured dogstatsd servers.
# The servers are specified as a comma separated list of host:port pairs. The
# default is to not send metrics to any dogstatsd servers. Generally
# DataDog recommends installing a local agent to collect metrics and forward
# them to the DataDog service. In this case the option value would be the
# local agent address: 127.0.0.1:8125.
#VGW_METRICS_DOGSTATS_SERVERS=

######################################
# VersityGW Backend Specific Options #
######################################

#########
# posix #
#########

# The posix backend translates S3 requests to file access in a local filesystem.
# The posix backend requires a filesystem that supports extended attributes.
# The top level directory for the gateway must be provided. All sub directories
# of the top level directory are treated as buckets, and all files/directories
# below the "bucket directory" are treated as the objects. The object
# name is split on "/" separator to translate to posix storage.
# For example:
# top level (VGW_BACKEND_ARG): /mnt/fs/gwroot
# bucket: mybucket
# object: a/b/c/myobject
# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject

# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
# change the ownership of newly created files and directories to the IAM
# account UID/GID.
#VGW_CHOWN_UID=false
#VGW_CHOWN_GID=false

# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false

# The default permissions mode when creating new directories is 0755. Use
# VGW_DIR_PERMS option to set a different mode for any new directory that the
# gateway creates. This applies to buckets created through the gateway as well
# as any parent directories automatically created with object uploads.
#VGW_DIR_PERMS=0755

# To enable object versions, the VGW_VERSIONING_DIR option must be set to the
# directory that will be used to store the object versions. The version
# directory must NOT be a subdirectory of the VGW_BACKEND_ARG directory.
#VGW_VERSIONING_DIR=

# The gateway uses xattrs to store metadata for objects by default. For systems
# that do not support xattrs, the VGW_META_SIDECAR option can be set to a
# directory that will be used to store the metadata for objects. This is
# currently experimental, and may have issues for some edge cases.
#VGW_META_SIDECAR=

# The VGW_META_NONE option will disable the metadata functionality for the
# gateway. This will cause the gateway to not store any metadata for objects
# or buckets. This include bucket ACLs and Policy. This may be useful for
# read only access to pre-existing data where the gateway should not modify
# the data. It is recommened to enable VGW_READ_ONLY (Global Options) along
# with this.
#VGW_META_NONE=false

# The gateway will use O_TMPFILE for writing objects while uploading and
# link the file to the final object name when the upload is complete if the
# filesystem supports O_TMPFILE. This creates an atomic object creation
# that is not visible to other clients or racing uploads until the upload
# is complete. This will not work if there is a different filesystem mounted
# below the bucket level than where the bucket resides. The VGW_DISABLE_OTMP
# option can be set to true to disable this functionality and force the fallback
# mode when O_TMPFILE is not available. This fallback will create a temporary
# file in the bucket directory and rename it to the final object name when
# the upload is complete if the final location is in the same filesystem, or
# copy the file to the final location if the final location is in a different
# filesystem. This fallback mode is still atomic, but may be less efficient
# than O_TMPFILE when the data needs to be copied into the final location.
#VGW_DISABLE_OTMP=false

###########
# scoutfs #
###########

# The scoutfs backend requires a ScoutFS filesystem type for the backend
# path. The object to posix name mappings follow the same rules as posix for
# scoutfs. The glacier mode functionality requires ScoutAM to be configured
# for tiering data from the ScoutFS filesystem to a mass stroage system.
# The mass storage system is often one or more tape libraries. Due to the
# high latency of tape, the glacier mode functionality is designed to
# give feedback to clients about object state and offer ability to request
# data to be staged back to disk without the client dealing with long
# request timeout settings.

# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
# GET object:  if file offline, return invalid object state
# HEAD object: if file offline, set obj storage class to GLACIER
#              if file offline and staging, x-amz-restore: ongoing-request="true"
#              if file offline and not staging, x-amz-restore: ongoing-request="false"
#              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
#              note: this expiry-date is not used but provided for client glacier compatibility
# ListObjects: if file offline, set obj storage class to GLACIER
# RestoreObject: add batch stage request to file
#VGW_SCOUTFS_GLACIER=false

# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
# change the ownership of newly created files and directories to the IAM
# account UID/GID.
#VGW_CHOWN_UID=false
#VGW_CHOWN_GID=false

# The VGW_SET_PROJECT_ID option will enable setting account defined ProjectID
# for newly created buckets, files, and directories if the account ProjectID
# is greater than 0 and the filesystem format version supports project IDs.
#VGW_SET_PROJECT_ID=false

# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false

# The default permissions mode when creating new directories is 0755. Use
# VGW_DIR_PERMS option to set a different mode for any new directory that the
# gateway creates. This applies to buckets created through the gateway as well
# as any parent directories automatically created with object uploads.
#VGW_DIR_PERMS=0755

# To enable object versions, the VGW_VERSIONING_DIR option must be set to the
# directory that will be used to store the object versions. The version
# directory must NOT be a subdirectory of the VGW_BACKEND_ARG directory.
# There may be implications for archive policy updates to include version
# directory as well. It is recommended to discuss archive implications of
# versioning with Versity support before enabling on an archiving filesystem.
#VGW_VERSIONING_DIR=

# The default behavior of the gateway is to automatically set the noarchive
# flag on the multipart upload parts while the multipart upload is in progress.
# This is to prevent the parts from being archived since they are temporary
# and will be deleted after the multipart upload is completed or aborted. The
# VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
#VGW_DISABLE_NOARCHIVE=false

######
# s3 #
######

# The s3 backend allows the gateway to forward requests to an S3 compatible
# service. This allows the gateway to act as a proxy for another S3 service.
# The backend S3 access is all done with a single configured account. The
# gateway will manage incoming multi-tenant access with the gateway configured
# IAM service. This gives stroage admins the ability to manage local gateway
# accounts while maintaining full control and a single account for the backend
# S3 service.

# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
#VGW_S3_ACCESS_KEY=
#VGW_S3_SECRET_KEY=
#VGW_S3_REGION=
#VGW_S3_ENDPOINT=
#VGW_S3_DISABLE_CHECKSUM=false
#VGW_S3_SSL_SKIP_VERIFY=false
#VGW_S3_DEBUG=false

########
# azure #
########

# The azure backend allows the gateway to store objects in Azure Blob Storage.
# Buckets created through the gateway map to blob containers within the
# configured storage account. This backend is useful when existing workflows
# expect an S3-compatible interface while data resides in Azure.

# When the azure backend is selected, configure credentials with one of the
# following approaches:
# - Shared key: Define AZ_ACCOUNT_NAME with the storage account name and
#   AZ_ACCESS_KEY with the corresponding account key.
# - SAS token: Set AZ_SAS_TOKEN to an account or container scoped SAS token.
#   Provide AZ_ENDPOINT if the token does not implicitly define the endpoint.
# - Default Azure credentials: Leave AZ_ACCOUNT_NAME and AZ_ACCESS_KEY blank
#   and configure the standard Azure identity environment variables supported
#   by the DefaultAzureCredential chain (e.g. AZURE_CLIENT_ID, AZURE_TENANT_ID,
#   AZURE_CLIENT_SECRET, managed identity, etc.).
# Use AZ_ENDPOINT to override the service URL (for example when targeting
# Azurite or a sovereign cloud). If unset, it defaults to
# https://<account>.blob.core.windows.net/ when an account name is provided.
#AZ_ACCOUNT_NAME=
#AZ_ACCESS_KEY=
#AZ_SAS_TOKEN=
#AZ_ENDPOINT=

##########
# plugin #
##########

# The plugin backend loads a Go plugin shared object that exposes a variable
# named "Backend" of type *plugins.BackendPlugin. The gateway uses the
# exported constructor to create the backend implementation at runtime.

# Set VGW_BACKEND_ARG to the absolute path of the compiled plugin (.so) file.
# The path must be readable by the gateway service account and remain stable
# across restarts.
#VGW_BACKEND_ARG=/usr/lib/versitygw/plugins/example.so

# Provide the plugin-specific configuration file path via VGW_PLUGIN_CONFIG.
# The gateway automatically forwards this value to the plugin backend when it
# starts up.
#VGW_PLUGIN_CONFIG=/etc/versitygw.d/example-plugin.conf