diff --git a/compose/.env b/compose/.env new file mode 100644 index 000000000..076ed05f7 --- /dev/null +++ b/compose/.env @@ -0,0 +1,61 @@ +## PostgreSQL related variables + +# Postgres Docker image +POSTGRES_IMAGE=library/postgres + +# Postgres user +POSTGRES_USER=postgres + +# Postgres password +POSTGRES_PASSWORD=magical_password + +# Postgres port number +POSTGRES_PORT=5432 + +# Postgres data directory +PGDATA=/data/postgres + + +## Logsearch related variables + +# Logsearch Docker image +LOGSEARCH_IMAGE=minio/logsearchapi:v4.0.2 + +# Logsearch storage max +LOGSEARCH_DISK_CAPACITY_GB=5 + +# Logsearch port number +LOGSEARCH_PORT=8080 + +# Log retention duration +LOGSEARCH_MAX_RETENTION_MONTHS=1 + +# Logsearch audit authentication token +LOGSEARCH_AUDIT_AUTH_TOKEN=c6rkqjZ03ElEUKQ7MtSeYBJ8q_p3GDFPBQAQJlcbBLA= + +# Logsearch query authentication token +LOGSEARCH_QUERY_AUTH_TOKEN=c6rkqjZ03ElEUKQ7MtSeYBJ8q_p3GDFPBQAQJlcbBLA= + + +## Console related variables + +# Console Docker image +CONSOLE_IMAGE=minio/console:v0.6.2 + +# Salt to encrypt JWT payload +CONSOLE_PBKDF_PASSPHRASE=top_secret + +# Required to encrypt JWT payload +CONSOLE_PBKDF_SALT=top_secret1 + +# MinIO Server URL +CONSOLE_MINIO_SERVER=http://localhost:9000 + + +## Prometheus related variables + +# Prometheus Docker image +PROMETHEUS_IMAGE=prom/prometheus:latest + +# Prometheus port number +PROMETHEUS_PORT=9999 diff --git a/compose/Readme.md b/compose/Readme.md new file mode 100644 index 000000000..f98c35edf --- /dev/null +++ b/compose/Readme.md @@ -0,0 +1,64 @@ + +## Console Docker Compose + +This compose file allows users to quickly deploy MinIO Console, LogSearch & Prometheus in a baremetal (non Kubernetes) environment. + +### Pre-requisites + +1. [MinIO](https://docs.minio.io/docs/distributed-minio-quickstart-guide.html) cluster up and running. +2. [mc](https://docs.minio.io/docs/minio-client-quickstart-guide.html) configured for this MinIO cluster. +3. [Docker-Compose](https://docs.docker.com/compose/) installed on the server. + +### Getting Started + +- Download the contents of `compose` directory on your machine. + +- Edit the `prometheus.yaml` file and fill in the correct target (MinIO Endpoint). Optionally setup the `bearer_token` as explained [here](https://github.com/minio/minio/tree/master/docs/metrics/prometheus#31-authenticated-prometheus-config). + +- Setup a console admin policy. + +```sh +cat > admin.json << EOF +{ + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "admin:*" + ], + "Effect": "Allow", + "Sid": "" + }, + { + "Action": [ + "s3:*" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::*" + ], + "Sid": "" + } + ] +} +EOF +``` + +Then create this policy on MinIO server: `mc admin policy add myminio consoleAdmin admin.json`. + +- Setup user and policy for Console + +``` +mc admin user add myminio console console123 +mc admin policy set myminio consoleAdmin user=console +``` + +- Configure Webhook target on the MinIO server. Remember to change the `token` value in below URL to the actual token value as set in the `.env` file. + +``` +mc admin config set myminio audit_webhook:1 endpoint=http://localhost:8080/api/ingest?token=c6rkqjZ03ElEUKQ7MtSeYBJ8q_p3GDFPBQAQJlcbBLA= +mc admin service restart myminio +``` + +### Configuration + +To configure the Console Compose file to custom setup, please take a look at the [`.env`](./.env) file. diff --git a/compose/docker-compose.yaml b/compose/docker-compose.yaml new file mode 100644 index 000000000..e4545f453 --- /dev/null +++ b/compose/docker-compose.yaml @@ -0,0 +1,62 @@ +version: '3.4' +services: + pg_database: + image: ${POSTGRES_IMAGE} + network_mode: host + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - PGDATA=${PGDATA} + - POSTGRES_DB=minio_logs + volumes: + - database:${PGDATA} + ports: + - ${POSTGRES_PORT}:${POSTGRES_PORT} + + log_search: + image: ${LOGSEARCH_IMAGE} + network_mode: host + environment: + - LOGSEARCH_AUDIT_AUTH_TOKEN=${LOGSEARCH_AUDIT_AUTH_TOKEN} + - LOGSEARCH_QUERY_AUTH_TOKEN=${LOGSEARCH_QUERY_AUTH_TOKEN} + - LOGSEARCH_DISK_CAPACITY_GB=${LOGSEARCH_DISK_CAPACITY_GB} + - LOGSEARCH_MAX_RETENTION_MONTHS=${LOGSEARCH_MAX_RETENTION_MONTHS} + - LOGSEARCH_PG_CONN_STR=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${POSTGRES_PORT}/minio_logs?sslmode=disable + ports: + - ${LOGSEARCH_PORT}:${LOGSEARCH_PORT} + command: ["/usr/bin/wait-for-it.sh", "localhost:${POSTGRES_PORT}", "--", "/logsearchapi"] + volumes: + - ./wait-for-it.sh:/usr/bin/wait-for-it.sh + depends_on: + - pg_database + + console: + image: ${CONSOLE_IMAGE} + network_mode: host + environment: + - CONSOLE_PBKDF_PASSPHRASE=${CONSOLE_PBKDF_PASSPHRASE} + - CONSOLE_PBKDF_SALT=${CONSOLE_PBKDF_SALT} + - LOGSEARCH_QUERY_AUTH_TOKEN=${LOGSEARCH_QUERY_AUTH_TOKEN} + - CONSOLE_MINIO_SERVER=${CONSOLE_MINIO_SERVER} + - CONSOLE_LOG_QUERY_URL=http://localhost:${LOGSEARCH_PORT} + - CONSOLE_PROMETHEUS_URL=http://localhost:${PROMETHEUS_PORT} + ports: + - "9090:9090" + command: server + depends_on: + - log_search + - prometheus + + prometheus: + image: ${PROMETHEUS_IMAGE} + network_mode: host + ports: + - ${PROMETHEUS_PORT}:${PROMETHEUS_PORT} + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.listen-address=:${PROMETHEUS_PORT} + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + +volumes: + database: diff --git a/compose/prometheus.yml b/compose/prometheus.yml new file mode 100644 index 000000000..0ee14fa7d --- /dev/null +++ b/compose/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 10s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 30s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: minio-node1 + metrics_path: /minio/v2/metrics/cluster + scheme: http + static_configs: + - targets: + - 'localhost:9000' diff --git a/compose/wait-for-it.sh b/compose/wait-for-it.sh new file mode 100755 index 000000000..3974640b0 --- /dev/null +++ b/compose/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file