From 68f29949a40163fb5b69ddb662057cb1a5dff3ed Mon Sep 17 00:00:00 2001 From: Michael Rennecke Date: Fri, 17 Mar 2017 22:57:44 +0100 Subject: [PATCH] simple monitoring solution with grafana, go-carbon and carbonapi --- .gitignore | 3 ++ carbonapi/Dockerfile | 21 +++++++++ configs/go-carbon.conf | 74 ++++++++++++++++++++++++++++++++ configs/storage-aggregation.conf | 29 +++++++++++++ configs/storage-schemas.conf | 16 +++++++ docker-compose.yml | 37 ++++++++++++++++ go-carbon/Dockerfile | 24 +++++++++++ 7 files changed, 204 insertions(+) create mode 100644 .gitignore create mode 100644 carbonapi/Dockerfile create mode 100644 configs/go-carbon.conf create mode 100644 configs/storage-aggregation.conf create mode 100644 configs/storage-schemas.conf create mode 100644 docker-compose.yml create mode 100644 go-carbon/Dockerfile diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4aad7b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +logs/ +whisper/ +grafana/ diff --git a/carbonapi/Dockerfile b/carbonapi/Dockerfile new file mode 100644 index 0000000..44ecf58 --- /dev/null +++ b/carbonapi/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.8-alpine + +RUN mkdir -p /go/src + +ADD https://github.com/dgryski/carbonapi/archive/master.zip /tmp/master.zip + +# build carbonapi +RUN set -x \ + && apk add --update git \ + && cd /go/src \ + && unzip /tmp/master.zip \ + && mv /go/src/carbonapi-master /go/src/carbonapi \ + && cd /go/src/carbonapi \ + && go-wrapper download \ + && go-wrapper install \ + && apk del git \ + && rm -f /tmp/master.zip \ + && rm -rf /var/cache/apk/* + +EXPOSE 8080 +ENTRYPOINT [ "/go/bin/carbonapi" ] diff --git a/configs/go-carbon.conf b/configs/go-carbon.conf new file mode 100644 index 0000000..15b818a --- /dev/null +++ b/configs/go-carbon.conf @@ -0,0 +1,74 @@ +[common] +user = "" +graph-prefix = "carbon.agents.{host}" +metric-endpoint = "local" +max-cpu = 2 +metric-interval = "1m0s" + +[whisper] +data-dir = "/data/graphite/whisper/" +schemas-file = "/etc/go-carbon/storage-schemas.conf" +aggregation-file = "" +workers = 2 +max-updates-per-second = 0 +sparse-create = false +enabled = true + +[cache] +max-size = 1000000 +write-strategy = "max" + +[udp] +listen = ":2003" +enabled = true +log-incomplete = false +buffer-size = 0 + +[tcp] +listen = ":2003" +enabled = true +buffer-size = 0 + +[pickle] +listen = ":2004" +max-message-size = 67108864 +enabled = true +buffer-size = 0 + +[carbonlink] +listen = ":7002" +enabled = true +read-timeout = "30s" + +[carbonserver] +listen = ":8080" +enabled = true +query-cache-enabled = true +query-cache-size-mb = 0 +find-cache-enabled = true +buckets = 10 +max-globs = 100 +metrics-as-counters = false +read-timeout = "1m0s" +idle-timeout = "1m0s" +write-timeout = "1m0s" +scan-frequency = "5m0s" + +[dump] +enabled = false +path = "" +restore-per-second = 0 + +[pprof] +listen = "localhost:7007" +enabled = false + +[[logging]] +logger = "" +file = "" +level = "info" +encoding = "mixed" +encoding-time = "iso8601" +encoding-duration = "seconds" + + diff --git a/configs/storage-aggregation.conf b/configs/storage-aggregation.conf new file mode 100644 index 0000000..bc5e1db --- /dev/null +++ b/configs/storage-aggregation.conf @@ -0,0 +1,29 @@ +[min] +pattern = \.lower$ +xFilesFactor = 0.1 +aggregationMethod = min + +[max] +pattern = \.upper(_\d+)?$ +xFilesFactor = 0.1 +aggregationMethod = max + +[sum] +pattern = \.sum$ +xFilesFactor = 0 +aggregationMethod = sum + +[count] +pattern = \.count$ +xFilesFactor = 0 +aggregationMethod = sum + +[count_legacy] +pattern = ^stats_counts.* +xFilesFactor = 0 +aggregationMethod = sum + +[default_average] +pattern = .* +xFilesFactor = 0.3 +aggregationMethod = average diff --git a/configs/storage-schemas.conf b/configs/storage-schemas.conf new file mode 100644 index 0000000..4f7f784 --- /dev/null +++ b/configs/storage-schemas.conf @@ -0,0 +1,16 @@ +# Documentation +# http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf + +# Carbon metrics are kept for 30 days in resolution of 10 seconds. +[carbon] +pattern = ^carbon\. +retentions = 10s:30d + +# User metrics are kept for: +# 24 hours in resolution of 5 seconds +# 30 days in resolution of 1 minute +# 1 year in resolution of 5 minutes +[user] +pattern = .* +retentions = 5s:24h,1m:30d,5m:1y + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b35d4d1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' +services: + go-carbon: + image: go-carbon + build: + context: ./go-carbon + hostname: go-carbon + container_name: go-carbon + command: -config=/etc/go-carbon/go-carbon.conf + volumes: + - ./configs:/etc/go-carbon + - ./whisper:/data/graphite/whisper/ + + carbonapi: + image: carbonapi + build: + context: ./carbonapi + hostname: carbonapi + container_name: carbonapi + command: -z=http://go-carbon:8080 -graphite=go-carbon:2003 -stdout + depends_on: + - go-carbon + + grafana: + image: grafana/grafana + hostname: grafana + container_name: grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - ./grafana:/var/lib/grafana + - ./logs:/var/log/grafana + depends_on: + - carbonapi + diff --git a/go-carbon/Dockerfile b/go-carbon/Dockerfile new file mode 100644 index 0000000..83deb36 --- /dev/null +++ b/go-carbon/Dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.8-alpine + +RUN mkdir -p /tmp/go-carbon + +ENV GOPATH /tmp/go-carbon/_vendor + +# build carbonapi +RUN set -x \ + && cd /tmp/go-carbon \ + && apk add --update git \ + && git clone https://github.com/lomik/go-carbon.git --depth=1 . \ + && git submodule init \ + && git submodule update --recursive \ + && go build github.com/lomik/go-carbon \ + && mv go-carbon /sbin/ \ + && cd / \ + && apk del git \ + && rm -f /tmp/$VERSION.zip \ + && rm -rf /tmp/go-carbon \ + && rm -rf /var/cache/apk/* + + +EXPOSE 2003 2004 7002 7007 2003/udp +ENTRYPOINT [ "/sbin/go-carbon" ]