commit 65cbe58f099582e7a2f78d7451a4ed8c348754e4
parent c3afc6121416c31b23f7b1cd8f913c9ad13a43c7
Author: Javier Sepulveda <javier.sepulveda@uv.es>
Date: Mon, 8 Jul 2024 13:03:30 +0200
Grafana Loki and Promtail, tutorials
Diffstat:
3 files changed, 266 insertions(+), 0 deletions(-)
diff --git a/system-administration/grafana-loki.rst b/system-administration/grafana-loki.rst
@@ -0,0 +1,136 @@
+..
+ This file is part of GNU TALER.
+ Copyright (C) 2014-2023 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU Affero General Public License as published by the Free Software
+ Foundation; either version 2.1, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+
+ @author Javier Sepulveda
+
+
+Grafana Loki
+############
+
+Loki is an aggregation system really similar to Prometheus, but instead of reading metrics, it reads logs (via push).
+
+.. contents:: Table of Contents
+ :depth: 1
+ :local:
+
+Create a system user
+====================
+
+.. code-block:: console
+
+ # useradd --system --no-create-home --shell /bin/false loki
+ # useradd --system --no-create-home --shell /bin/false promtail
+
+Download Loki
+=============
+
+* Download
+* Extract
+* Copy to /usr/local/bin
+* Set ownership and permissions
+
+.. code-block:: console
+
+ # cd /tmp
+ # wget https://github.com/grafana/loki/releases/download/v3.0.0/loki-linux-amd64.zip
+ # unzip loki-linux-amd64.zip -d .
+ # mv loki-linux-amd64 loki
+ # cp loki /usr/local/bin/
+
+Loki configuration file
+=========================================
+
+.. code-block:: yaml
+
+ auth_enabled: false
+
+ server:
+ http_listen_port: 3100
+ grpc_listen_port: 9096
+
+ common:
+ instance_addr: 127.0.0.1
+ path_prefix: /tmp/loki
+ storage:
+ filesystem:
+ chunks_directory: /tmp/loki/chunks
+ rules_directory: /tmp/loki/rules
+ replication_factor: 1
+ ring:
+ kvstore:
+ store: inmemory
+
+ query_range:
+ results_cache:
+ cache:
+ embedded_cache:
+ enabled: true
+ max_size_mb: 100
+
+ schema_config:
+ configs:
+ - from: 2020-10-24
+ store: tsdb
+ object_store: filesystem
+ schema: v13
+ index:
+ prefix: index_
+ period: 24h
+
+Systemd service file
+====================
+
+.. code-block:: systemd
+
+ # Path: /etc/systemd/system/loki.service
+
+ [Unit]
+ Description=Loki service
+ After=network.target
+
+ [Service]
+ Type=simple
+ User=loki
+ ExecStart=/usr/local/bin/loki -config.file /etc/loki/config.yml
+ # Give a reasonable amount of time for the server to start up/shut down
+ TimeoutSec = 120
+ Restart = on-failure
+ RestartSec = 2
+
+ [Install]
+ WantedBy=multi-user.target
+
+Refresh systemd and restart
+===========================
+
+.. code-block:: console
+
+ # systemctl daemon-reload
+ # systemctl enable --now loki
+ # systemctl status loki
+ # systemctl restart prometheus
+
+Check
+-----
+
+http://147.87.255.218:3100
+
+Grafana control panel
+=====================
+
+You can now to go the `Grafana control panel <a href="https://grafana.taler.net">`_ and easily
+add the new Loki data source.
+
+
diff --git a/system-administration/grafana-promtail.rst b/system-administration/grafana-promtail.rst
@@ -0,0 +1,128 @@
+..
+ This file is part of GNU TALER.
+ Copyright (C) 2014-2023 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU Affero General Public License as published by the Free Software
+ Foundation; either version 2.1, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+
+ @author Javier Sepulveda
+
+Grafana Promtail
+################
+
+Promtail can read two different types of logs. A regular log file, or information from a systemd journal unit.
+
+.. contents:: Table of Contents
+ :depth: 1
+ :local:
+
+
+Download promtail
+=================
+
+* Download
+* Extract
+* Copy to /usr/bin
+* Set ownership and permissions
+
+.. code-block::
+
+ # cd /tmp
+ # wget https://github.com/grafana/loki/releases/download/v3.0.0/promtail-linux-amd64.zip
+ # unzip promtail-linux-amd64.zip -d .
+ # mv promtail-linux-amd64 promtail
+ # cp promtail /usr/local/bin
+
+Create system user
+==================
+
+.. code-block:: console
+
+ # useradd --system --no-create-home --shell /bin/false promtail
+ # usermod -G adm -a promtail # Add the "promtail" user, to the "adm" group if necessary.
+
+Promtail configuration file
+===========================
+
+.. code-block:: yaml
+
+ # Path: /etc/promtail/config.yml
+
+ server:
+ http_listen_port: 9080
+ grpc_listen_port: 0
+
+ positions:
+ filename: /tmp/positions.yaml
+
+ clients:
+ - url: http://localhost:3100/loki/api/v1/push
+
+ scrape_configs:
+ - job_name: system
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: nginx
+ __path__: /var/log/nginx/*log
+
+
+Promtail systemd service file
+=============================
+
+.. code-block:: systemd
+
+ # Path: /etc/systemd/system/promtail.service
+
+ [Unit]
+ Description=Promtail service
+ After=network.target
+
+ [Service]
+ Type=simple
+ User=promtail
+ ExecStart=/usr/local/bin/promtail -config.file /etc/promtail/config.yml
+ # Give a reasonable amount of time for promtail to start up/shut down
+ TimeoutSec = 60
+ Restart = on-failure
+ RestartSec = 2
+
+ [Install]
+ WantedBy=multi-user.target
+
+
+Start and enable Promtail
+-------------------------
+
+.. code-block:: console
+
+ # systemctl enable --now promtail.service
+ # systemctl status promtail.service
+
+
+Promtail temporary files
+========================
+
+.. code-block:: console
+
+ # cd /tmp
+ # touch positions.yaml
+ # chown promtail:promtail /tmp/positions.yaml
+
+Grafana control panel
+=====================
+
+To check if Promtail is reading properly either your log files or the journald units, you can go to the "Explore" section in the Grafana
+control panel, choose the right Loki connector, choose your desired log file or journald unit, and execute the query.
+If that works, you can convert this temporary query into a real Grafana "dashboard", to continue working afterwards with additional filtering options.
+
+
diff --git a/system-administration/index.rst b/system-administration/index.rst
@@ -29,3 +29,5 @@ System Administration tutorials
nginx-prometheus-exporter
prometheus-node-exporter
prometheus-postgres-exporter
+ grafana-loki
+ grafana-promtail
