feat: add loki functionality to transport logs (#1479)

* Updates the config from env files to include metrics options

* updates README

* update loki transport

* update readme

* fix port number on metrics app

* update query params metrics

* fix import sidecarConfig

* fix readme

* fix .env and readme

Author: filvecchiato

.env.local

@@ -3,4 +3,4 @@
 # For more information on how to use .env files and environment variables
 # consult the Configuration section in the README.
 
-SAS_SUBSTRATE_URL=ws://127.0.0.1:9944
+SAS_SUBSTRATE_URL=ws://127.0.0.1:9944

README.md

@@ -170,6 +170,14 @@ For more information on our configuration manager visit its readme [here](https:
 - `SAS_SUBSTRATE_URL`: URL to which the RPC proxy will attempt to connect to, defaults to
     `ws://127.0.0.1:9944`. Accepts both a websocket, and http URL.
 
+### Metrics Server
+
+- `SAS_METRICS_ENABLED`: Boolean to enable the metrics server instance with Prometheus (server metrics) and Loki (logging) connections. Defaults to false.
+- `SAS_METRICS_PROM_HOST`: The host of the prometheus server used to listen to metrics emitted, defaults to `127.0.0.1`.
+- `SAS_METRICS_PROM_PORT`: The port of the prometheus server, defaults to `9100`.
+- `SAS_METRICS_LOKI_HOST`: The host of the loki server used to pull the logs, defaults to `127.0.0.1`.
+- `SAS_METRICS_LOKI_PORT`: The port of the loki server, defaults to `3100`
+
 #### Custom substrate types
 
 Some chains require custom type definitions in order for Sidecar to know how to decode the data
@@ -239,23 +247,10 @@ file you can `symlink` it with `.env.test`. For example you could run
 commands `ln` and `unlink` for more info.)
 
 ### Prometheus server
-Prometheus metrics can be enabled by running sidecar with the following flag :
-
-```bash
-yarn start --prometheus
-```
-
-You can also define a custom port by running :
-
-```bash
-yarn start --prometheus --prometheus-port=<YOUR_CUSTOM_PORT>
-```
 
-You can also expand the metrics tracking capabilities to include query params by running:
+Prometheus metrics can be enabled by running sidecar with the following env configuration: `SAS_METRICS_ENABLED`=true
 
-```bash
-yarn start --prometheus --prometheus-queryparams
-```
+You can also expand the metrics tracking capabilities to include query params by adding to the env configuration: `SAS_METRICS_INCLUDE_QUERYPARAMS`=true
 
 The metrics endpoint can then be accessed :
 - on the default port : `http://127.0.0.1:9100/metrics` or
@@ -279,7 +274,17 @@ The blocks controller also includes the following route-specific metrics:
 - `sas_extrinsics_per_block`: type histogram and tracks the returned extrinsics per block
 - `sas_seconds_per_block`: type histogram and tracks the request time per block
 
-The metrics registry is injected in the Response object when the `-prometheus` flag is selected, allowing to extend the controller based metrics to any given controller from within the controller functions.
+The metrics registry is injected in the Response object when the `SAS_METRICS_ENABLED` flag is set to `true` in the `.env` file, allowing to extend the controller based metrics to any given controller from within the controller functions.
+
+To successfully run and access the metrics and logs in Grafana (for example) the following are required:
+
+- prometheus server (info [here](https://prometheus.io/docs/prometheus/latest/getting_started/))
+- loki server and promtail (info [here](https://grafana.com/docs/loki/latest/setup/install/))
+
+For mac users using homebrew:
+```bash
+brew install prometheus loki promtail
+```
 
 ## Debugging fee and staking payout calculations
 

package.json

@@ -62,7 +62,8 @@
     "lru-cache": "^10.4.3",
     "prom-client": "^15.1.3",
     "rxjs": "^7.8.1",
-    "winston": "^3.14.1"
+    "winston": "^3.14.1",
+    "winston-loki": "^6.1.2"
   },
   "devDependencies": {
     "@substrate/dev": "^0.7.1",

src/SidecarConfig.ts

@@ -1,4 +1,4 @@
-// Copyright 2017-2022 Parity Technologies (UK) Ltd.
+// Copyright 2017-2024 Parity Technologies (UK) Ltd.
 // This file is part of Substrate API Sidecar.
 //
 // Substrate API Sidecar is free software: you can redistribute it and/or modify
@@ -72,6 +72,14 @@ export class SidecarConfig {
 				WRITE_MAX_FILE_SIZE: config.Get(MODULES.LOG, CONFIG.WRITE_MAX_FILE_SIZE) as number,
 				WRITE_MAX_FILES: config.Get(MODULES.LOG, CONFIG.WRITE_MAX_FILES) as number,
 			},
+			METRICS: {
+				ENABLED: config.Get(MODULES.METRICS, CONFIG.ENABLED) as boolean,
+				PROM_HOST: config.Get(MODULES.METRICS, CONFIG.PROM_HOST) as string,
+				PROM_PORT: config.Get(MODULES.METRICS, CONFIG.PROM_PORT) as number,
+				LOKI_HOST: config.Get(MODULES.METRICS, CONFIG.LOKI_HOST) as string,
+				LOKI_PORT: config.Get(MODULES.METRICS, CONFIG.LOKI_PORT) as number,
+				INCLUDE_QUERYPARAMS: config.Get(MODULES.METRICS, CONFIG.INCLUDE_QUERYPARAMS) as boolean,
+			},
 		};
 
 		return this._config;

src/Specs.ts

@@ -1,4 +1,4 @@
-// Copyright 2017-2022 Parity Technologies (UK) Ltd.
+// Copyright 2017-2024 Parity Technologies (UK) Ltd.
 // This file is part of Substrate API Sidecar.
 //
 // Substrate API Sidecar is free software: you can redistribute it and/or modify
@@ -32,6 +32,7 @@ export class Specs {
 		this.appendLogSpecs();
 		this.appendSubstrateSpecs();
 		this.appendExpressSpecs();
+		this.appendMetricsSpecs();
 
 		return this._specs;
 	}
@@ -242,4 +243,60 @@ export class Specs {
 			}),
 		);
 	}
+
+	private static appendMetricsSpecs() {
+		if (!this._specs) {
+			throw APPEND_SPEC_ERROR;
+		}
+
+		this._specs.appendSpec(
+			MODULES.METRICS,
+			this._specs.getSpec(CONFIG.ENABLED, 'Whether or not to enable metrics', {
+				default: 'false',
+				type: 'boolean',
+				regexp: /^true|false$/,
+			}),
+		);
+
+		this._specs.appendSpec(
+			MODULES.METRICS,
+			this._specs.getSpec(CONFIG.PROM_HOST, 'Prometheus host', {
+				default: '127.0.0.1',
+				type: 'string',
+			}),
+		);
+
+		this._specs.appendSpec(
+			MODULES.METRICS,
+			this._specs.getSpec(CONFIG.PROM_PORT, 'Prometheus port', {
+				default: 9100,
+				type: 'number',
+				regexp: /^\d{2,6}$/,
+			}),
+		);
+
+		this._specs.appendSpec(
+			MODULES.METRICS,
+			this._specs.getSpec(CONFIG.LOKI_HOST, 'Loki host', {
+				default: '127.0.0.1',
+				type: 'string',
+			}),
+		);
+		this._specs.appendSpec(
+			MODULES.METRICS,
+			this._specs.getSpec(CONFIG.LOKI_PORT, 'Loki port', {
+				default: 3100,
+				type: 'number',
+			}),
+		);
+
+		this._specs.appendSpec(
+			MODULES.METRICS,
+			this._specs.getSpec(CONFIG.INCLUDE_QUERYPARAMS, 'Include query params in the labels of the metrics', {
+				default: 'false',
+				type: 'boolean',
+				regexp: /^true|false$/,
+			}),
+		);
+	}
 }

src/logging/Log.ts

@@ -16,6 +16,7 @@
 
 import { createLogger, Logger } from 'winston';
 import { ConsoleTransportInstance, FileTransportInstance } from 'winston/lib/winston/transports';
+import LokiTransport from 'winston-loki';
 
 import { SidecarConfig } from '../SidecarConfig';
 import { consoleTransport, fileTransport } from './transports';
@@ -24,7 +25,7 @@ import { consoleTransport, fileTransport } from './transports';
  * Access a singleton winston.Logger that will be intialized on first use.
  */
 export class Log {
-	private static _transports: (ConsoleTransportInstance | FileTransportInstance)[] | undefined;
+	private static _transports: (ConsoleTransportInstance | FileTransportInstance | LokiTransport)[] | undefined;
 	private static _logger: Logger | undefined;
 	private static create(): Logger {
 		if (this._logger) {
@@ -40,6 +41,16 @@ export class Log {
 			this._transports.push(fileTransport('logs.log'));
 		}
 
+		if (SidecarConfig.config.METRICS.ENABLED) {
+			this._transports.push(
+				new LokiTransport({
+					host: `http://${SidecarConfig.config.METRICS.LOKI_HOST}:${SidecarConfig.config.METRICS.LOKI_PORT}`,
+					useWinstonMetaAsLabels: true,
+					json: true,
+				}),
+			);
+		}
+
 		this._logger = createLogger({
 			transports: this._transports,
 			exitOnError: false,

src/main.ts

@@ -66,11 +66,12 @@ async function main() {
 	startUpPrompt(config.SUBSTRATE.URL, chainName.toString(), implName.toString());
 
 	const preMiddlewares = [json(), middleware.httpLoggerCreate(logger)];
-	if (args.prometheus) {
+
+	if (config.METRICS.ENABLED) {
 		// Create Metrics App
 		const metricsApp = new MetricsApp({
-			port: 9100,
-			host: config.EXPRESS.HOST,
+			port: config.METRICS.PROM_PORT,
+			host: config.METRICS.PROM_HOST,
 		});
 
 		// Generate metrics middleware

src/metrics/Metrics.ts

@@ -3,7 +3,7 @@ import { Application, Request, Response } from 'express';
 import client from 'prom-client';
 
 import { Log } from '../logging/Log';
-import { parseArgs } from '../parseArgs';
+import { SidecarConfig } from '../SidecarConfig';
 import { IMetric, MetricType } from '../types/metrics';
 import { config } from '.';
 
@@ -35,11 +35,9 @@ export default class Metrics_App {
 	/**
 	 * @param appConfig configuration for app.
 	 */
-	constructor({ host }: IAppConfiguration) {
-		const args = parseArgs();
-
-		this.includeQueryParams = !!args.prometheus_queryparams;
-		this.port = Number(args.prometheus_port);
+	constructor({ host, port }: IAppConfiguration) {
+		this.includeQueryParams = SidecarConfig.config.METRICS.INCLUDE_QUERYPARAMS;
+		this.port = port;
 		this.app = express();
 		this.host = host;
 		this.registry = new client.Registry();

src/parseArgs.ts

@@ -23,19 +23,6 @@ export const parseArgs = (): Namespace => {
 		action: 'store_true',
 		help: 'print substrate-api-sidecar version',
 	});
-	parser.add_argument('-p', '--prometheus', {
-		action: 'store_true',
-		help: 'enable the prometheus metrics endpoint',
-	});
-	parser.add_argument('-pp', '--prometheus-port', {
-		type: 'int',
-		default: 9100,
-		help: 'specify the port number on which the prometheus metrics are exposed [default: 9100]',
-	});
-	parser.add_argument('-pq', '--prometheus-queryparams', {
-		action: 'store_true',
-		help: 'enambles query parameters in the prometheus metrics',
-	});
 
 	return parser.parse_args() as Namespace;
 };

src/types/sidecar-config/CONFIG.ts

@@ -34,4 +34,10 @@ export enum CONFIG {
 	WRITE_PATH = 'WRITE_PATH',
 	WRITE_MAX_FILE_SIZE = 'WRITE_MAX_FILE_SIZE',
 	WRITE_MAX_FILES = 'WRITE_MAX_FILES',
+	ENABLED = 'ENABLED',
+	LOKI_HOST = 'LOKI_HOST',
+	PROM_PORT = 'PROM_PORT',
+	PROM_HOST = 'PROM_HOST',
+	LOKI_PORT = 'LOKI_PORT',
+	INCLUDE_QUERYPARAMS = 'INCLUDE_QUERYPARAMS',
 }