diff --git a/.gitignore b/.gitignore index 56287790..1b84a66d 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ tests/!s3cfg.local.default # patches *.patch + +# grafana's local database (kept on filesystem for survival between instantiations) +metrics-exploration/grafana_data/** \ No newline at end of file diff --git a/docker-compose-metrics.yml b/docker-compose-metrics.yml new file mode 100644 index 00000000..7efefd05 --- /dev/null +++ b/docker-compose-metrics.yml @@ -0,0 +1,64 @@ +services: + telegraf: + image: telegraf + container_name: telegraf + restart: always + volumes: + - ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + - influxdb + links: + - influxdb + ports: + - '8125:8125/udp' + + influxdb: + image: influxdb + container_name: influxdb + restart: always + environment: + - DOCKER_INFLUXDB_INIT_MODE=setup + - DOCKER_INFLUXDB_INIT_USERNAME=admin + - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass + - DOCKER_INFLUXDB_INIT_ORG=myorg + - DOCKER_INFLUXDB_INIT_BUCKET=metrics + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token + ports: + - '8086:8086' + volumes: + - influxdb_data:/var/lib/influxdb + + grafana: + image: grafana/grafana + container_name: grafana-server + restart: always + depends_on: + - influxdb + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_INSTALL_PLUGINS= + links: + - influxdb + ports: + - '3000:3000' + volumes: + - ./metrics-exploration/grafana_data:/var/lib/grafana + - ./metrics-exploration/provisioning:/etc/grafana/provisioning + + versitygw: + image: versity/versitygw:latest + container_name: versitygw + ports: + - "7070:7070" + environment: + - ROOT_ACCESS_KEY=user + - ROOT_SECRET_KEY=password + - VGW_METRICS_STATSD_SERVERS=telegraf:8125 + depends_on: + - telegraf + command: > + posix /tmp/vgw + +volumes: + influxdb_data: {} diff --git a/metrics-exploration/README.md b/metrics-exploration/README.md new file mode 100644 index 00000000..57c57df5 --- /dev/null +++ b/metrics-exploration/README.md @@ -0,0 +1,19 @@ +# Versity Gateway Dashboard + +This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana. + +The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb. + +This implementation uses the influxql query language. + +## Usage + +From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack. + +To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`. + +The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is. + +The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with username: `admin` and password: `admin`. + +To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual. \ No newline at end of file diff --git a/metrics-exploration/aws_env_setup.sh b/metrics-exploration/aws_env_setup.sh new file mode 100644 index 00000000..d3ef2a9e --- /dev/null +++ b/metrics-exploration/aws_env_setup.sh @@ -0,0 +1,4 @@ +export AWS_SECRET_ACCESS_KEY=password +export AWS_ACCESS_KEY_ID=user +export AWS_ENDPOINT_URL=http://127.0.0.1:7070 +export AWS_REGION=us-east-1 diff --git a/metrics-exploration/docker-compose.yml b/metrics-exploration/docker-compose.yml new file mode 100644 index 00000000..eecf45cb --- /dev/null +++ b/metrics-exploration/docker-compose.yml @@ -0,0 +1,64 @@ +services: + telegraf: + image: telegraf + container_name: telegraf + restart: always + volumes: + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + - influxdb + links: + - influxdb + ports: + - '8125:8125/udp' + + influxdb: + image: influxdb + container_name: influxdb + restart: always + environment: + - DOCKER_INFLUXDB_INIT_MODE=setup + - DOCKER_INFLUXDB_INIT_USERNAME=admin + - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass + - DOCKER_INFLUXDB_INIT_ORG=myorg + - DOCKER_INFLUXDB_INIT_BUCKET=metrics + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token + ports: + - '8086:8086' + volumes: + - influxdb_data:/var/lib/influxdb + + grafana: + image: grafana/grafana + container_name: grafana-server + restart: always + depends_on: + - influxdb + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_INSTALL_PLUGINS= + links: + - influxdb + ports: + - '3000:3000' + volumes: + - ./grafana_data:/var/lib/grafana + - ./provisioning:/etc/grafana/provisioning + + versitygw: + image: versity/versitygw:latest + container_name: versitygw + ports: + - "7070:7070" + environment: + - ROOT_ACCESS_KEY=user + - ROOT_SECRET_KEY=password + - VGW_METRICS_STATSD_SERVERS=telegraf:8125 + depends_on: + - telegraf + command: > + posix /tmp/vgw + +volumes: + influxdb_data: {} diff --git a/metrics-exploration/provisioning/dashboards/influxql.yaml b/metrics-exploration/provisioning/dashboards/influxql.yaml new file mode 100644 index 00000000..25d96ae7 --- /dev/null +++ b/metrics-exploration/provisioning/dashboards/influxql.yaml @@ -0,0 +1,25 @@ +apiVersion: 1 + +providers: + # an unique provider name. Required + - name: 'influxql' + # Org id. Default to 1 + orgId: 1 + # name of the dashboard folder. + folder: 'influxql' + # folder UID. will be automatically generated if not specified + folderUid: '' + # provider type. Default to 'file' + type: file + # disable dashboard deletion + disableDeletion: false + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: true + options: + # path to dashboard files on disk. Required when using the 'file' type + path: /etc/grafana/provisioning/dashboards/influxql + # use folder names from filesystem to create folders in Grafana + foldersFromFilesStructure: true + diff --git a/metrics-exploration/provisioning/dashboards/influxql/Versity S3 Gateway.json b/metrics-exploration/provisioning/dashboards/influxql/Versity S3 Gateway.json new file mode 100644 index 00000000..0888f318 --- /dev/null +++ b/metrics-exploration/provisioning/dashboards/influxql/Versity S3 Gateway.json @@ -0,0 +1,1538 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "iconColor": "red", + "name": "Gateway Annotations" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 1, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "versitygw_sucess_CompleteMultipartUpload", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "CompleteMultipartUpload" + } + ] + }, + { + "alias": "versitygw_sucess_AbortMultipartUpload", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "AbortMultipartUpload" + } + ] + }, + { + "alias": "versitygw_sucess_CreateBucket", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "CreateBucket" + } + ] + }, + { + "alias": "versitygw_sucess_CreateMultipartUpload", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "CreateMultipartUpload" + } + ] + }, + { + "alias": "versitygw_sucess_DeleteObject", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "DeleteObject" + } + ] + }, + { + "alias": "versitygw_sucess_GetObject", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "GetObject" + } + ] + }, + { + "alias": "versitygw_sucess_HeadObject", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "HeadObject" + } + ] + }, + { + "alias": "versitygw_sucess_ListObjectsV2", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "ListObjectsV2" + } + ] + }, + { + "alias": "versitygw_sucess_PutObject", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "PutObject" + } + ] + }, + { + "alias": "versitygw_sucess_UploadPart", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_success_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "UploadPart" + } + ] + } + ], + "title": "Success Sum by Action", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "alias": "versitygw_failed_createBucket", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "versitygw_failed_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "condition": "AND", + "key": "action::tag", + "operator": "=", + "value": "CreateBucket" + } + ] + }, + { + "alias": "versitygw_failed_deleteObject", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_failed_count", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"value\") FROM \"versitygw_failed_count\" WHERE (\"action\"::tag = 'DeleteObject') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "DeleteObject" + } + ] + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_failed_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "ActionUnDetected" + } + ] + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_failed_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "DeleteBucket" + } + ] + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_failed_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "HeadObject" + } + ] + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_failed_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "UploadPart" + } + ] + } + ], + "title": "Fail Sum by Action", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "versitygw_bytes_written", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "title": "Bytes Written Sum", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "versitygw_bytes_read", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "title": "Bytes Read Sum", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "versitygw_object_created_CompleteMultipartUpload", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "versitygw_object_created_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "CompleteMultipartUpload" + } + ] + }, + { + "alias": "versitygw_object_created_PutObject", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "versitygw_object_created_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "PutObject" + } + ] + } + ], + "title": "Object Created Sum", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "versitygw_object_removed_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "action::tag", + "operator": "=", + "value": "DeleteObject" + } + ] + } + ], + "title": "Object Removed Sum", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Versity S3 Gateway", + "uid": "cdupnlwv714hsd", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/metrics-exploration/provisioning/datasources/influxdb.yml b/metrics-exploration/provisioning/datasources/influxdb.yml new file mode 100644 index 00000000..dc3dfdb3 --- /dev/null +++ b/metrics-exploration/provisioning/datasources/influxdb.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +datasources: + - name: influxdb + type: influxdb + isDefault: true + access: proxy + url: http://influxdb:8086 + jsonData: + dbName: 'metrics' + httpHeaderName1: 'Authorization' + secureJsonData: + httpHeaderValue1: 'Token my-super-secret-auth-token' diff --git a/metrics-exploration/telegraf.conf b/metrics-exploration/telegraf.conf new file mode 100644 index 00000000..e41b47b1 --- /dev/null +++ b/metrics-exploration/telegraf.conf @@ -0,0 +1,34 @@ +[global_tags] + +[agent] + debug = true + quiet = false + interval = "60s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + hostname = "versitygw" + omit_hostname = false + +[[outputs.file]] + files = ["stdout"] + +[[outputs.influxdb_v2]] + urls = ["http://influxdb:8086"] + timeout = "5s" + token = "my-super-secret-auth-token" + organization = "myorg" + bucket = "metrics" + +[[inputs.statsd]] + protocol = "udp4" + service_address = ":8125" + percentiles = [90] + metric_separator = "_" + datadog_extensions = false + allowed_pending_messages = 10000 + percentile_limit = 1000 diff --git a/metrics-exploration/test.sh b/metrics-exploration/test.sh new file mode 100644 index 00000000..7af95c1a --- /dev/null +++ b/metrics-exploration/test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +. ./aws_env_setup.sh + +aws s3 mb s3://test +aws s3 cp docker-compose.yml s3://test/test.yaml