diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 6f6f3af..26da514 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -100,6 +100,27 @@ nfpms: rpm: group: "System Environment/Daemons" + # RPM specific scripts. + scripts: + # The pretrans script runs before all RPM package transactions / stages. + #pretrans: ./extra/pretrans.sh + # The posttrans script runs after all RPM package transactions / stages. + posttrans: ./extra/posttrans.sh + + contents: + - src: extra/versitygw@.service + dst: /lib/systemd/system/versitygw@.service + + - src: extra/example.conf + dst: /etc/versitygw.d/example.conf + type: config + + - dst: /etc/versitygw.d + type: dir + file_info: + mode: 0700 + + # yaml-language-server: $schema=https://goreleaser.com/static/schema.json # vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 0b448c9..9cdeedb 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -197,13 +197,13 @@ func initFlags() []cli.Flag { &cli.StringFlag{ Name: "access-log", Usage: "enable server access logging to specified file", - EnvVars: []string{"LOGFILE"}, + EnvVars: []string{"LOGFILE", "VGW_ACCESS_LOG"}, Destination: &accessLog, }, &cli.StringFlag{ Name: "log-webhook-url", Usage: "webhook url to send the audit logs", - EnvVars: []string{"WEBHOOK"}, + EnvVars: []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"}, Destination: &logWebhookURL, }, &cli.StringFlag{ diff --git a/cmd/versitygw/s3.go b/cmd/versitygw/s3.go index 2339e95..a5407ae 100644 --- a/cmd/versitygw/s3.go +++ b/cmd/versitygw/s3.go @@ -44,6 +44,7 @@ to an s3 storage backend service.`, Usage: "s3 proxy server access key id", Value: "", Required: true, + EnvVars: []string{"VGW_S3_ACCESS_KEY"}, Destination: &s3proxyAccess, Aliases: []string{"a"}, }, @@ -52,6 +53,7 @@ to an s3 storage backend service.`, Usage: "s3 proxy server secret access key", Value: "", Required: true, + EnvVars: []string{"VGW_S3_SECRET_KEY"}, Destination: &s3proxySecret, Aliases: []string{"s"}, }, @@ -59,23 +61,27 @@ to an s3 storage backend service.`, Name: "endpoint", Usage: "s3 service endpoint, default AWS if not specified", Value: "", + EnvVars: []string{"VGW_S3_ENDPOINT"}, Destination: &s3proxyEndpoint, }, &cli.StringFlag{ Name: "region", Usage: "s3 service region, default 'us-east-1' if not specified", Value: "us-east-1", + EnvVars: []string{"VGW_S3_REGION"}, Destination: &s3proxyRegion, }, &cli.BoolFlag{ Name: "disable-checksum", Usage: "disable gateway to server object checksums", Value: false, + EnvVars: []string{"VGW_S3_DISABLE_CHECKSUM"}, Destination: &s3proxyDisableChecksum, }, &cli.BoolFlag{ Name: "ssl-skip-verify", Usage: "skip ssl cert verification for s3 service", + EnvVars: []string{"VGW_S3_SSL_SKIP_VERIFY"}, Value: false, Destination: &s3proxySslSkipVerify, }, @@ -83,6 +89,7 @@ to an s3 storage backend service.`, Name: "debug", Usage: "output extra debug tracing", Value: false, + EnvVars: []string{"VGW_S3_DEBUG"}, Destination: &s3proxyDebug, }, }, diff --git a/cmd/versitygw/scoutfs.go b/cmd/versitygw/scoutfs.go index b7b1d80..65041de 100644 --- a/cmd/versitygw/scoutfs.go +++ b/cmd/versitygw/scoutfs.go @@ -48,6 +48,7 @@ move interfaces as well as support for tiered filesystems.`, Name: "glacier", Usage: "enable glacier emulation mode", Aliases: []string{"g"}, + EnvVars: []string{"VGW_SCOUTFS_GLACIER"}, Destination: &glacier, }, }, diff --git a/extra/example.conf b/extra/example.conf new file mode 100644 index 0000000..c930446 --- /dev/null +++ b/extra/example.conf @@ -0,0 +1,272 @@ +################################### +# VersityGW systemd configuration # +################################### + +# Copy this file to /etc/versitygw.d/ and rename it to a unique service name. +# For example, if the service name is "mygateway", then the file should be +# named /etc/versitygw.d/mygateway.conf. +# The systemd template file /lib/systemd/system/versitygw@.service will +# automatically load the configuration file for the service name. +# To start the gateway, use the following command: +# systemctl start versitygw@mygateway +# To enable the gateway to start on boot, use the following command: +# systemctl enable versitygw@mygateway +# To stop the gateway, use the following command: +# systemctl stop versitygw@mygateway + +# There can be multiple gateway services running on the same host. Each +# gateway service must have a unique service name with a unique configuration +# file in /etc/versitygw.d/. They must also listen on different ports and/or +# interfaces using the VGW_PORT option. + +############################## +# VersityGW Required Options # +############################## + +# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3 +# This defines the backend that the VGW will use for data access. +VGW_BACKEND=posix + +# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined +# as the the top level directory for the gateway. +# All sub directories of the top level directory are treated as buckets, +# and all files/directories below the "bucket directory" are treated as +# the objects. The object name is split on "/" separator to translate +# to posix storage. +# For example: +# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot +# bucket: mybucket +# object: a/b/c/myobject +# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject +VGW_BACKEND_ARG= + +############################ +# VersityGW Global Options # +############################ + +# commented options are the default values + +# The following must be set, and do not have default values +# The access and secret options will specify the root account credentials. +# The root account is granted full authorization to all API requests after +# authentication. +ROOT_ACCESS_KEY_ID= +ROOT_SECRET_ACCESS_KEY= + +# The following are optional, and have the default values as listed + +# The VGW_PORT option will specify the listening port for the S3 server. +# This option can use either the form : which will listen only +# on the network interface that matches the IP on the specified port, or +# : which will listen on all network interfaces on the specified port. +# The spec can either be IP dotted notation or a resolvable hostname. +# The spec can either be a numeric port or the service name typically +# in /etc/services. +#VGW_PORT=:7070 + +# The VGW_REGION option will specify the region that the S3 server will +# report to clients. This option is optional, and defaults to "us-east-1". +#VGW_REGION=us-east-1 + +# The VGW_CERT and VGW_KEY options will specify the SSL certificate and +# private key that the S3 server will use for SSL connections. This option +# is optional, and defaults to not using SSL. +#VGW_CERT= +#VGW_KEY= + +# The VGW_ADMIN_PORT option will specify the listening port for the admin +# server. The admin server endpoint can optionally be set to listen on a +# different interface or port than the S3 service. This allows for better +# control of firewall restrictions to the admin endpoint. The certs for this +# can be different certs than specified for the S3 service. The default when +# these are not specified is to have the admin server listen on the same +# endpoint as the S3 service. +# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin +# server will use SSL. +#VGW_ADMIN_PORT= +#VGW_ADMIN_CERT= +#VGW_ADMIN_CERT_KEY= + +# The VGW_QUIET option when set will supress the S3 server request summary +# logging to stdout. +#VGW_QUIET=false + +# The VGW_HEALTH option when set will specify the URL to accept health checks +# on. The health check endpoint is often used for load balancers to verify +# gateway is alive. The health endpoint masks any bucket with this setting. +# For example, if the health endpoint is set to /health, the gateway will not +# allow creating or listing contents of a bucket called "health". The health +# endpoint is unauthenticated, and returns a 200 status for GET. +#VGW_HEALTH= + +############### +# Access Logs # +############### + +# The VGW_ACCESS_LOG option when set will specify the file to log all S3 +# server requests to. This option is optional, and defaults to not logging. +# It is suggested to use absolute paths for the server log file because the +# server may chdir into the backend root directory and change locations for +# relative paths. +# The log file format follows the AWS S3 access log format documented in +# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html. +#VGW_ACCESS_LOG= + +# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the +# S3 server request access logs to. The access logs are JSON encoded when +# sent to the webhook. +#VGW_LOG_WEBHOOK_URL= + +############## +# Event Logs # +############## + +# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL, +# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all +# bucket events will be sent to the kafka service. The gateway events are +# similar to AWS S3 events, and are documented in the wiki: +# https://github.com/versity/versitygw/wiki/Events-Notifications. +#VGW_EVENT_KAFKA_URL= +#VGW_EVENT_KAFKA_TOPIC= +#VGW_EVENT_KAFKA_KEY= + +# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL +# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the +# the NATS messaging service. The gateway events are similar to AWS S3 events, +# and are documented in the wiki: +# https://github.com/versity/versitygw/wiki/Events-Notifications. +#VGW_EVENT_NATS_URL= +#VGW_EVENT_NATS_TOPIC= + +# The VGW_DEBUG option enables verbose debug log output to stdout. This output +# includes details for signature verification steps. This is generally only +# useful for debugging the S3 server, and should not be used in production. +#VGW_DEBUG=false + +################ +# IAM services # +################ + +# The VGW_IAM_DIR option will enable the internal IAM service with accounts +# stored in a file under the specified directory. This is provided to minimize +# dependencies on outside services for basic functionality. The local account +# files are plain text and only protected with file permissions. This IAM +# service is added for convenience, but is not considered as secure or scalable +# as a dedicated IAM service. +#VGW_IAM_DIR= + +# The ldap options will enable the LDAP IAM service with accounts stored in an +# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR, +# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access, +# secret credentials and role respectively. The other options are used to +# connect to the LDAP service. +#VGW_IAM_LDAP_URL= +#VGW_IAM_LDAP_BASE_DN= +#VGW_IAM_LDAP_BIND_DN= +#VGW_IAM_LDAP_BIND_PASS= +#VGW_IAM_LDAP_QUERY_BASE= +#VGW_IAM_LDAP_OBJECT_CLASSES= +#VGW_IAM_LDAP_ACCESS_ATR= +#VGW_IAM_LDAP_SECRET_ATR= +#VGW_IAM_LDAP_ROLE_ATR= + +# The VGW_S3 IAM service is similar to the internal IAM service, but instead +# stores the account information JSON encoded in an S3 object. This should use +# a bucket that is not accessible to general users when using s3 backend to +# prevent access to account credentials. This IAM service is added for +# convenience, but is not considered as secure or scalable as a dedicated IAM +# service. +#VGW_S3_IAM_ACCESS_KEY= +#VGW_S3_IAM_SECRET_KEY= +#VGW_S3_IAM_REGION= +#VGW_S3_IAM_ENDPOINT= +#VGW_S3_IAM_BUCKET= +#VGW_S3_IAM_NO_VERIFY= + +############### +# IAM caching # +############### + +# The IAM cache is intended to ease the load on the IAM service and increase +# the Gateway performance by caching accounts and credentials for the TTL time +# interval. Disabling this will cause a request to the configured IAM service +# for each incoming request to retrieve the corresponding account credentials. +# The cache is enabled by default. The TTL specifies how long to cache +# credentials, and the prune value determines the interval for expired entries +# to be removed from the cache. Increasing the TTL may lessen the load on the +# IAM service backend, but may have out of date account info until the next +# interval. Increasing the prune value may reduce memory use at the cost of +# added CPU to check cache expirations. +#VGW_IAM_CACHE_DISABLE=false +#VGW_IAM_CACHE_TTL=120 +#VGW_IAM_CACHE_PRUNE=3600 + +###################################### +# VersityGW Backend Specific Options # +###################################### + +######### +# posix # +######### + +# The posix backend translates S3 requests to file access in a local filesystem. +# The posix backend requires a filesystem that supports extended attributes. +# The top level directory for the gateway must be provided. All sub directories +# of the top level directory are treated as buckets, and all files/directories +# below the "bucket directory" are treated as the objects. The object +# name is split on "/" separator to translate to posix storage. +# For example: +# top level: /mnt/fs/gwroot +# bucket: mybucket +# object: a/b/c/myobject +# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject + +# There are currently no further options other than VGW_BACKEND_ARG for the +# posix backend. + +########### +# scoutfs # +########### + +# The scoutfs backend requires a ScoutFS filesystem type for the backend +# path. The glacier mode functionality requires ScoutAM to be configured +# for tiering data from the ScoutFS filesystem to a mass stroage system. +# The mass storage system is often one or more tape libraries. Due to the +# high latency of tape, the glacier mode functionality is designed to +# give feedback to clients about object state and offer ability to request +# data to be staged back to disk without the client dealing with long +# request timeout settings. + +# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior. +# GET object: if file offline, return invalid object state +# HEAD object: if file offline, set obj storage class to GLACIER +# if file offline and staging, x-amz-restore: ongoing-request="true" +# if file offline and not staging, x-amz-restore: ongoing-request="false" +# if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT" +# note: this expiry-date is not used but provided for client glacier compatibility +# ListObjects: if file offline, set obj storage class to GLACIER +# RestoreObject: add batch stage request to file +#VGW_SCOUTFS_GLACIER=false + +###### +# s3 # +###### + +# The s3 backend allows the gateway to forward requests to an S3 compatible +# service. This allows the gateway to act as a proxy for another S3 service. +# The backend S3 access is all done with a single configured account. The +# gateway will manage incoming multi-tenant access with the gateway configured +# IAM service. This gives stroage admins the ability to manage local gateway +# accounts while maintaining full control and a single account for the backend +# S3 service. + +# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must +# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will +# default to "us-east-1" and "https://s3.amazonaws.com" respectively. +#VGW_S3_ACCESS_KEY= +#VGW_S3_SECRET_KEY= +#VGW_S3_REGION= +#VGW_S3_ENDPOINT= +#VGW_S3_DISABLE_CHECKSUM=false +#VGW_S3_SSL_SKIP_VERIFY=false +#VGW_S3_DEBUG=false diff --git a/extra/posttrans.sh b/extra/posttrans.sh new file mode 100644 index 0000000..f7b1bb6 --- /dev/null +++ b/extra/posttrans.sh @@ -0,0 +1,2 @@ +#!/bin/bash +systemctl daemon-reload diff --git a/extra/versitygw@.service b/extra/versitygw@.service new file mode 100644 index 0000000..d2947d3 --- /dev/null +++ b/extra/versitygw@.service @@ -0,0 +1,33 @@ +[Unit] +Description=VersityGW +Documentation=https://github.com/versity/versitygw/wiki +Wants=network-online.target +After=network-online.target remote-fs.target +AssertFileIsExecutable=/usr/bin/versitygw +AssertPathExists=/etc/versitygw.d/%i.conf + +[Service] +WorkingDirectory=/root +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=versitygw-%i + +User=root +Group=root + +EnvironmentFile=/etc/versitygw.d/%i.conf + +ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"' + +# Let systemd restart this service always +Restart=always + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=65536 + +# Specifies the maximum number of threads this process can create +TasksMax=infinity + +[Install] +WantedBy=multi-user.target +