Big refactoring of the repository

.gitignore

@@ -1 +1,2 @@
 data
+.env

Makefile

@@ -0,0 +1,26 @@
+.PHONY: clean
+
+export UID := $(shell id -u)
+export GID := $(shell id -g)
+
+run: prepare
+	docker compose up
+
+daemon: prepare
+	docker compose up -d
+
+stop:
+	docker compose stop
+
+clean:
+	docker compose down --remove-orphans
+	git clean -fxd data/
+	rm -f .env
+
+.env:
+	@echo UID=$(UID) > $@
+	@echo GID=$(GID) >> $@
+
+reset: clean .env
+
+prepare: .env

README.md

@@ -2,16 +2,20 @@
 ```sh
 git clone https://github.com/lomik/graphite-clickhouse-tldr
 cd graphite-clickhouse-tldr
-export UID
-docker-compose up
+make
+# or
+make daemon
+make stop # to stop the docker
 ```
-Open http://127.0.0.1/ in browser
+Open http://127.0.0.1/ in browser, credentials are `admin:tldr`
 
 ### Mapped Ports
 
 Host | Container | Service
 ---- | --------- | -------------------------------------------------------------------------------------------------------------------
-  80 |        80 | [nginx](https://www.nginx.com/resources/admin-guide/)
+  80 |      3000 | [grafana](https://grafana.com/grafana)
 2003 |      2003 | [carbon receiver - plaintext](http://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol)
 2004 |      2004 | [carbon receiver - pickle](http://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-pickle-protocol)
 2006 |      2006 | [carbon receiver - prometheus remote write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_write%3E)
+9090 |      9090 | carbonapi/graphite-web endpoint
+9092 |      9092 | prometheus UI endpoint

carbon-clickhouse.conf

@@ -1,12 +1,12 @@
 [common]
 # Prefix for store all internal carbon-clickhouse graphs. Supported macroses: {host}
-metric-prefix = "carbon.agents.{host}"
+metric-prefix = "carbon.{host}"
 # Endpoint for store internal carbon metrics. Valid values: "" or "local", "tcp://host:port", "udp://host:port"
 metric-endpoint = "local"
 # Interval of storing internal metrics. Like CARBON_METRIC_INTERVAL
 metric-interval = "1m0s"
 # GOMAXPROCS
-max-cpu = 1
+max-cpu = 10
 
 [logging]
 # "stderr", "stdout" can be used as file name
@@ -18,6 +18,7 @@ level = "warn"
 # Folder for buffering received data
 path = "/data/carbon-clickhouse/"
 # Rotate (and upload) file interval.
+chunk-max-size = 400000000
 # Minimize chunk-interval for minimize lag between point receive and store
 chunk-interval = "1s"
 # Auto-increase chunk interval if the number of unprocessed files is grown
@@ -30,30 +31,30 @@ chunk-auto-interval = "5:5s,10:60s"
 type = "points"
 table = "graphite_data"
 threads = 1
-url = "http://clickhouse:8123/"
-timeout = "1m0s"
+url = "http://clickhouse:8123/?http_receive_timeout=3600&receive_timeout=3600&http_send_timeout=3600&send_timeout=3600"
+timeout = "4m0s"
 
 [upload.graphite_reverse]
 type = "points-reverse"
 table = "graphite_reverse"
 threads = 1
-url = "http://clickhouse:8123/"
-timeout = "1m0s"
+url = "http://clickhouse:8123/?http_receive_timeout=3600&receive_timeout=3600&http_send_timeout=3600&send_timeout=3600"
+timeout = "4m0s"
 
 [upload.graphite_index]
 type = "index"
 table = "graphite_index"
 threads = 1
-url = "http://clickhouse:8123/"
-timeout = "1m0s"
+url = "http://clickhouse:8123/?http_receive_timeout=3600&receive_timeout=3600&http_send_timeout=3600&send_timeout=3600"
+timeout = "4m0s"
 cache-ttl = "12h0m0s"
 
 [upload.graphite_tagged]
 type = "tagged"
 table = "graphite_tagged"
 threads = 1
-url = "http://clickhouse:8123/"
-timeout = "1m0s"
+url = "http://clickhouse:8123/?http_receive_timeout=3600&receive_timeout=3600&http_send_timeout=3600&send_timeout=3600"
+timeout = "4m0s"
 cache-ttl = "12h0m0s"
 
 [udp]

carbonapi.yml

@@ -0,0 +1,132 @@
+# Need to be URL, http or https
+# This url specifies the backend or a loadbalancer
+#
+# If you are using carbonzipper you should set it to
+# zipper's url
+#
+# If you are using plain go-carbon or graphite-clickhouse
+# you should set it to URL of go-carbon's carbonserver module
+# or graphite-clickhouse's http url.
+# Listen address, should always include hostname or ip address and a port.
+listen: ":7080"
+# Controls headers that would be passed to the backend
+headersToPass:
+  - "X-Dashboard-Id"
+  - "X-Grafana-Org-Id"
+  - "X-Panel-Id"
+# Max concurrent requests to CarbonZipper
+concurency: 1000
+cache:
+  # Type of caching. Valid: "mem", "memcache", "null"
+  type: "mem"
+  # Cache limit in megabytes
+  size_mb: 0
+  # Default cache timeout value. Identical to DEFAULT_CACHE_DURATION in graphite-web.
+  defaultTimeoutSec: 60
+# Amount of CPUs to use. 0 - unlimited
+cpus: 0
+# Timezone, default - local
+tz: ""
+
+# By default, functions like aggregate inherit tags from first series (for compatibility with graphite-web)
+# If set to true, tags are extracted from seriesByTag arguments
+#extractTagsFromArgs: false
+maxBatchSize: 0
+graphite:
+  # Host:port where to send internal metrics
+  # Empty = disabled
+  host: "carbon-clickhouse:2003"
+  interval: "60s"
+  prefix: "carbonapi"
+  # rules on how to construct metric name. For now only {prefix} and {fqdn} is supported.
+  # {prefix} will be replaced with the content of {prefix}
+  # {fqdn} will be repalced with fqdn
+  pattern: "{prefix}.{fqdn}"
+# Maximium idle connections to carbonzipper
+idleConnections: 1000
+pidFile: ""
+# See https://github.com/go-graphite/carbonzipper/blob/master/example.conf#L70-L108 for format explanation
+upstreams:
+  # Number of 100ms buckets to track request distribution in. Used to build
+  # 'carbon.zipper.hostname.requests_in_0ms_to_100ms' metric and friends.
+  # Requests beyond the last bucket are logged as slow (default of 10 implies
+  # "slow" is >1 second).
+  # The last bucket is _not_ called 'requests_in_Xms_to_inf' on purpose, so
+  # we can change our minds about how many buckets we want to have and have
+  # their names remain consistent.
+  buckets: 10
+  timeouts:
+    find: "10s"
+    render: "50s"
+    connect: "200ms"
+  # Number of concurrent requests to any given backend - default is no limit.
+  # If set, you likely want >= MaxIdleConnsPerHost
+  concurrencyLimitPerServer: 1000
+
+  # Configures how often keep alive packets will be sent out
+  keepAliveInterval: "30s"
+
+  # Control http.MaxIdleConnsPerHost. Large values can lead to more idle
+  # connections on the backend servers which may bump into limits; tune with care.
+  maxIdleConnsPerHost: 1000
+  backendsv2:
+    backends:
+      - groupName: "clickhouse-cluster1"
+        # supported:
+        #    carbonapi_v2_pb - carbonapi 0.11 or earlier version of protocol.
+        #    carbonapi_v3_pb - new protocol, http interface (native)
+        #    carbonapi_v3_grpc - new protocol, gRPC interface (native)
+        #    protobuf, pb, pb3 - same as carbonapi_v2_pb
+        #    msgpack - protocol used by graphite-web 1.1 and metrictank
+        #    auto - carbonapi will do it's best to guess if it's carbonapi_v3_pb or carbonapi_v2_pb
+        #
+        #  non-native protocols will be internally converted to new protocol, which will increase memory consumption
+        protocol: "carbonapi_v3_pb"
+        # supported:
+        #    "broadcast" - send request to all backends in group and merge responses. This was default behavior for carbonapi 0.11 or earlier
+        #    "roundrobin" - send request to one backend.
+        #    "all - same as "broadcast"
+        #    "rr" - same as "roundrobin"
+        lbMethod: "rr"
+        # amount of retries in case of unsuccessful request
+        maxTries: 3
+        # amount of metrics per fetch request. Default: 0 - unlimited. If not specified, global will be used
+        maxBatchSize: 0
+        # interval for keep-alive http packets. If not specified, global will be used
+        keepAliveInterval: "30s"
+        # override for global concurrencyLimit.
+        concurrencyLimit: 1000
+        # override for global maxIdleConnsPerHost
+        maxIdleConnsPerHost: 1000
+        # per-group timeout override. If not specified, global will be used.
+        # Please note that ONLY min(global, local) will be used.
+        timeouts:
+          # Maximum backend request time for find requests.
+          find: "10s"
+          # Maximum backend request time for render requests. This is total one and doesn't take into account in-flight requests.
+          render: "50s"
+          # Timeout to connect to the server
+          connect: "200ms"
+        servers:
+          - "http://graphite-clickhouse:9090"
+# If not zero, enabled cache for find requests
+# This parameter controls when it will expire (in seconds)
+# Default: 600 (10 minutes)
+graphTemplates: /etc/graphTemplates.yaml
+expireDelaySec: 10
+# Uncomment this to get the behavior of graphite-web as proposed in https://github.com/graphite-project/graphite-web/pull/2239
+# Beware this will make darkbackground graphs less readable
+# defaultColors:
+#   "red": "ff0000"
+#   "green": "00ff00"
+#   "blue": "#0000ff"
+#   "darkred": "#c80032"
+#   "darkgreen": "00c800"
+#   "darkblue": "002173"
+logger:
+  - logger: ""
+    file: "stderr"
+    level: "debug"
+    encoding: "console"
+    encodingTime: "iso8601"
+    encodingDuration: "seconds"

docker-compose.yaml

@@ -1,44 +1,66 @@
-version: "3.5"
+---
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:21.8
-    user: "${UID:?You must do 'export UID' to launch}"
+    image: clickhouse/clickhouse-server:24.8
+    user: "${UID:?run `make prepare`}:${GID:?run `make prepare`}"
     volumes:
-    - "./rollup.xml:/etc/clickhouse-server/config.d/rollup.xml"
-    - "./init.sql:/docker-entrypoint-initdb.d/init.sql"
-    - "./data/clickhouse:/var/lib/clickhouse"
+      - "./rollup.xml:/etc/clickhouse-server/config.d/rollup.xml"
+      - "./init.sql:/docker-entrypoint-initdb.d/init.sql"
+      - "./data/clickhouse:/var/lib/clickhouse"
+      - "./data/clickhouse:/var/log/clickhouse-server"
     networks:
       - graphite-clickhouse-tldr
+    environment:
+      CLICKHOUSE_SKIP_USER_SETUP: 1
+    cap_add: &caps
+      - SYS_PTRACE
+      - NET_ADMIN
+      - IPC_LOCK
+      - SYS_NICE
   carbon-clickhouse:
-    image: ghcr.io/lomik/carbon-clickhouse:0.11.2
-    user: "${UID:?You must do 'export UID' to launch}"
+    image: ghcr.io/go-graphite/carbon-clickhouse:0.11.8
+    user: "${UID:?run `make prepare`}:${GID:?run `make prepare`}"
     volumes:
-    - "./data:/data"
-    - "./carbon-clickhouse.conf:/etc/carbon-clickhouse/carbon-clickhouse.conf"
+      - "./data:/data"
+      - "./carbon-clickhouse.conf:/etc/carbon-clickhouse/carbon-clickhouse.conf"
     ports:
-    - "2003:2003" # plain tcp
-    - "2003:2003/udp" # plain udp
-    - "2004:2004" # pickle
-    - "2006:2006" # prometheus remote write
+      - "2003:2003"  # plain tcp
+      - "2003:2003/udp"  # plain udp
+      - "2004:2004"  # pickle
+      - "2006:2006"  # prometheus remote write
     networks:
       - graphite-clickhouse-tldr
   graphite-clickhouse:
-    image: ghcr.io/lomik/graphite-clickhouse:0.13.2
+    image: ghcr.io/go-graphite/graphite-clickhouse:0.14.0
     volumes:
-    - "./rollup.xml:/etc/graphite-clickhouse/rollup.xml"
-    - "./graphite-clickhouse.conf:/etc/graphite-clickhouse/graphite-clickhouse.conf"
+      - "./rollup.xml:/etc/graphite-clickhouse/rollup.xml"
+      - "./graphite-clickhouse.conf:/etc/graphite-clickhouse/graphite-clickhouse.conf"
+    ports:
+      - "9090:9090"  # carbonapi/graphite-web endpoint
+      - "9092:9092"  # prometheus endpoint
     networks:
       - graphite-clickhouse-tldr
-  graphite-web:
-    image: graphiteapp/graphite-statsd:1.1.8-1
+  carbonapi:
+    image: ghcr.io/go-graphite/carbonapi:0.17.0
+    volumes:
+      - "./carbonapi.yml:/etc/carbonapi.yml"
+      - "./graphTemplates.yaml:/etc/graphTemplates.yaml"
+    networks:
+      - graphite-clickhouse-tldr
+    ports:
+      - "7080:7080"  # http ui
+  grafana:
+    user: "${UID:?run `make prepare`}:${GID:?run `make prepare`}"
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: "tldr"
+    image: grafana/grafana:11.3.0-ubuntu
     volumes:
-    - "./graphiteweb_entrypoint.sh:/entrypoint.sh"
-    - "./graphiteweb.conf:/opt/graphite/webapp/graphite/local_settings.py"
-    entrypoint: /entrypoint.sh
+      - "./data/grafana:/var/lib/grafana"
+      - "./grafana-graphite-datasource.yaml:/etc/grafana/provisioning/datasources/graphite.yaml"
     networks:
       - graphite-clickhouse-tldr
     ports:
-    - "80:80" # http ui
+      - "80:3000"  # http ui
 
 networks:
   graphite-clickhouse-tldr:

grafana-graphite-datasource.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Graphite
+    type: graphite
+    access: proxy
+    url: http://carbonapi:7080
+    jsonData:
+      graphiteVersion: '1.1'
+