commit 3c92b8e9b52b726aa0f21255114b9743b3fc458d
parent c152abe5a430f1d2810a98beb1384487df84ed21
Author: Florian Dold <dold@taler.net>
Date: Fri, 11 Sep 2026 14:54:13 +0200
sysadmin docs: describe the Sentol monitoring architecture
Outline metrics collection, journal forwarding, service checks and
alert delivery, with a Mermaid diagram and links to the web interfaces.
Refer to tsys-infra.git/monitoring for operational details and retain
Uptime Kuma documentation. Remove the obsolete monitoring guides.
Add the Sphinx Mermaid extension to the Debian and Nix dependencies.
Diffstat:
12 files changed, 107 insertions(+), 1248 deletions(-)
diff --git a/README b/README
@@ -6,6 +6,7 @@ To build things on a Debian (-based) system, install these packages:
- python3-myst-parser
- python3-sphinx-book-theme
- python3-sphinx-design
+ - python3-sphinxcontrib-mermaid
- graphviz
- texlive-latex-extra
- dvipng
@@ -15,6 +16,11 @@ Then, do "make html" for HTML, "make texinfo" for Texinfo, etc.
The output is in subdir ‘_build’.
+Mermaid diagrams use sphinxcontrib-mermaid (packaged in Debian 13 trixie).
+HTML output renders them in the browser using JavaScript from the extension's
+default CDN. Mermaid CLI is not needed. Put diagrams inside an "only:: html"
+directive and include an equivalent prose description for other output formats.
+
Branch ‘prebuilt’ is special. Its contents (or a subset) are used as
submodules in other Git repos (e.g., Exchange, Merchant, Sync).
diff --git a/conf.py b/conf.py
@@ -59,6 +59,7 @@ extensions = [
"sphinx.ext.graphviz",
"sphinx.ext.imgmath",
"sphinxcontrib.plantuml",
+ "sphinxcontrib.mermaid",
"httpdomain.httpdomain",
"myst_parser",
"sphinx_multitoc_numbering",
@@ -74,6 +75,10 @@ imgmath_use_preview = True
plantuml = os.environ.get("PLANTUML", "plantuml")
plantuml_output_format = "svg"
+# Render HTML Mermaid diagrams in the browser using the extension's CDN assets.
+# Keep diagrams in ``only:: html`` blocks, with prose for other output formats.
+mermaid_output_format = "raw"
+
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
diff --git a/flake.nix b/flake.nix
@@ -35,6 +35,7 @@
python-pkgs.myst-parser
python-pkgs.sphinxcontrib-httpdomain
python-pkgs.sphinxcontrib-plantuml
+ python-pkgs.sphinxcontrib-mermaid
python-pkgs.sphinx-design
]))
];
@@ -64,6 +65,7 @@
python-pkgs.myst-parser
python-pkgs.sphinxcontrib-httpdomain
python-pkgs.sphinxcontrib-plantuml
+ python-pkgs.sphinxcontrib-mermaid
python-pkgs.sphinx-design
]))
];
diff --git a/system-administration/grafana-loki.rst b/system-administration/grafana-loki.rst
@@ -1,207 +0,0 @@
-..
- This file is part of GNU TALER.
- Copyright (C) 2014-2023 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU Affero General Public License as published by the Free Software
- Foundation; either version 3.0, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
-
- @author Javier Sepulveda
-
-
-Grafana Loki
-############
-
-Loki is an aggregation system really similar to Prometheus, but instead of reading metrics, it reads logs (via push).
-Please check the `official documentation website <https://grafana.com/docs/loki/latest/get-started/>`_ for additional information.
-
-Create a system user
-====================
-
-.. code-block:: console
-
- # useradd --system --no-create-home --shell /bin/false loki
- # useradd --system --no-create-home --shell /bin/false promtail
-
-Download Loki
-=============
-
-* Download
-* Extract
-* Copy to /usr/local/bin
-* Set ownership and permissions
-
-.. code-block:: console
-
- # cd /tmp
- # wget https://github.com/grafana/loki/releases/download/v3.0.0/loki-linux-amd64.zip
- # unzip loki-linux-amd64.zip -d .
- # mv loki-linux-amd64 loki
- # cp loki /usr/local/bin/
-
-Loki configuration file
-=======================
-
-.. code-block:: yaml
-
- auth_enabled: false
-
- server:
- http_listen_port: 3100
- grpc_listen_port: 9096
-
- common:
- instance_addr: 127.0.0.1
- path_prefix: /tmp/loki
- storage:
- filesystem:
- chunks_directory: /tmp/loki/chunks
- rules_directory: /tmp/loki/rules
- replication_factor: 1
- ring:
- kvstore:
- store: inmemory
-
- query_range:
- results_cache:
- cache:
- embedded_cache:
- enabled: true
- max_size_mb: 100
-
- schema_config:
- configs:
- - from: 2020-10-24
- store: tsdb
- object_store: filesystem
- schema: v13
- index:
- prefix: index_
- period: 24h
-
-Systemd service file
-====================
-
-.. code-block:: systemd
-
- # Path: /etc/systemd/system/loki.service
-
- [Unit]
- Description=Loki service
- After=network.target
-
- [Service]
- Type=simple
- User=loki
- ExecStart=/usr/local/bin/loki -config.file /etc/loki/config.yml
- # Give a reasonable amount of time for the server to start up/shut down
- TimeoutSec = 120
- Restart = on-failure
- RestartSec = 2
-
- [Install]
- WantedBy=multi-user.target
-
-Refresh systemd and restart
-===========================
-
-.. code-block:: console
-
- # systemctl daemon-reload
- # systemctl enable --now loki
- # systemctl status loki
- # systemctl restart prometheus
-
-Check
------
-
-http://ip:3100
-
-Close the 3100 port with Nginx
-==============================
-
-.. code-block:: nginx
-
- # Path: /etc/nginx/sites-available/loki.conf
-
- upstream loki {
- server 127.0.0.1:3100; # Loopback
- keepalive 15;
- }
-
- server {
- listen 80;
- listen [::]:80;
-
- server_name loki.taler-ops.ch;
- root /dev/null;
-
- # LOKI
-
- location / {
- proxy_read_timeout 1800s;
- proxy_connect_timeout 1600s;
- proxy_pass http://loki;
- }
-
- location /ready {
- proxy_pass http://loki;
- proxy_http_version 1.1;
- proxy_set_header Connection "Keep-Alive";
- proxy_set_header Proxy-Connection "Keep-Alive";
- proxy_redirect off;
- auth_basic "off";
- }
-
- }
-
-- Enable in Nginx the new virtualhost file
-
-.. code-block:: console
-
- # ln -s /etc/nginx/sites-available/loki.conf /etc/nginx/sites-enabled/loki.conf
- # nginx -t
- # systemctl reload nginx
-
-Edit the loki configuration file
-================================
-
-.. code-block:: yaml
-
- #Path: /etc/loki/config.yml
- server:
- http_listen_port: 3100
- grpc_listen_port: 9096
- # Add this to close the 3100 port
- http_listen_address: 127.0.0.1
-
-Refresh systemd and restart
-===========================
-
-.. code-block:: console
-
- # systemctl restart nginx
- # systemctl restart loki
- # systemctl restart prometheus
-
-Check
------
-
-Check that the 3100 port is publicly closed by typing in your Web browser http://ip:3100. At
-the same time change through the Grafana control panel, your loki data source server URL field,
-so it can connect now through the specified subdomain in the nginx virtualhost file (in our test case
-loki.taler-ops.ch).
-
-Grafana control panel
-=====================
-
-You can now to go the `Grafana control panel <https://grafana.taler.net>`_ and easily
-add the new Loki data source.
-
diff --git a/system-administration/index.rst b/system-administration/index.rst
@@ -22,15 +22,9 @@ Internal System Administration
:maxdepth: 1
:glob:
- uptime-kuma
taler-monitoring-infrastructure
+ uptime-kuma
backups
support
- prometheus
- prometheus-alerts
- nginx-prometheus-exporter
- prometheus-node-exporter
- prometheus-postgres-exporter
- grafana-loki
debian-repos
ci-debian-versions
diff --git a/system-administration/nginx-prometheus-exporter.rst b/system-administration/nginx-prometheus-exporter.rst
@@ -1,182 +0,0 @@
-..
- This file is part of GNU TALER.
- Copyright (C) 2014-2023 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU Affero General Public License as published by the Free Software
- Foundation; either version 3.0, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
-
- @author Javier Sepulveda
-
-
-Nginx Prometheus Exporter
-#########################
-
-Download nginx-prometheus-exporter
-==================================
-
-* Download
-* Extract
-* Copy to /usr/local/bin
-* Set ownership and permissions
-
-.. code-block:: console
-
- # mkdir -p /tmp/nginx_exporter_tmp && cd /tmp/nginx_exporter_tmp
- # wget https://github.com/nginxinc/nginx-prometheus-exporter/releases/download/v1.2.0/nginx-prometheus-exporter_1.2.0_linux_amd64.tar.gz
- # tar axf nginx-prometheus-exporter_1.2.0_linux_amd64.tar.gz
- # cd nginx-prometheus-exporter
- # cp nginx-prometheus-exporter /usr/local/bin
-
-
-Create a system user
-====================
-
-.. code-block:: console
-
- # useradd --system --no-create-home --shell /bin/false nginx-exporter
-
-
-Systemd configuration
-=====================
-
-Nginx-prometheus-exporter systemd service file
-----------------------------------------------
-
-.. code-block:: systemd
-
- # Path:/etc/systemd/system/nginx-exporter.service
-
- [Unit]
- Description=NGINX Prometheus Exporter
- Requires=nginx-exporter.socket
-
- [Service]
- User=nginx-exporter
- ExecStart=/usr/local/bin/nginx-prometheus-exporter --nginx.scrape-uri="http://127.0.0.1/metrics" --web.systemd-socket
-
- [Install]
- WantedBy=multi-user.target
-
-
-Nginx-prometheus-exporter systemd socket file
----------------------------------------------
-
-.. code-block:: systemd
-
- # Path: /etc/systemd/system/nginx-exporter.socket
-
- [Unit]
- Description=NGINX Prometheus Exporter
-
- [Socket]
- ListenStream=9113
-
- [Install]
- WantedBy=sockets.target
-
-Modify the Prometheus configuration file
-========================================
-
-.. code-block:: yaml
-
- # Path: /etc/prometheus/prometheus.yml
-
- # Job, for nginx_exporter
- - job_name: 'nginx_exporter'
- static_configs:
- - targets: ['localhost:9113'] # choose port number
-
-
-Nginx configuration
-===================
-
-.. code-block:: nginx
-
- server {
- listen localhost;
- root /var/www/html;
- index index.html;
- server_name _;
-
- location /metrics {
- stub_status on;
- access_log off;
- allow 127.0.0.1;
- deny all;
- }
-
- }
-
-
-.. code-block:: console
-
- ## Create the Nginx symbolic link to enable the virtualhost file
- # ln -s /etc/nginx/sites-available/nginx-exporter.conf /etc/nginx/sites-enabled/nginx-exporter.conf
- # nginx -t
- # systemctl reload nginx
- ## Check with curl
- # curl http://localhost/metrics
-
-Restart everything
-==================
-
-.. code-block:: console
-
- # systemctl daemon-reload # Only necessary if modified
- # systemctl enable --now node-exporter.service
- # systemctl status node-exporter.service
- # systemctl restart prometheus.service
-
-
-Close the 9113 port with Nginx
-==============================
-
-* Change the nginx-exporter.socket file
-
-.. code-block:: console
-
- # Path: /etc/systemd/system/nginx-exporter.socket
-
- [Unit]
- Description=NGINX Prometheus Exporter
-
- # Add here the loopback ip address
- [Socket]
- ListenStream=127.0.0.1:9113
-
- [Install]
- WantedBy=sockets.target
-
-
-Add a new server block to Nginx
-===============================
-
-.. code-block:: nginx
-
- server {
- listen 80;
- listen [::]:80;
-
- server_name ip_address | subdomain.ltd;
- location /nginx-exporter {
- proxy_pass http://localhost:9113;
- }
-
- }
-
-Create a new dashboard in Grafana
-=================================
-
-You can now go to the `Grafana dashboards <a href="https://grafana.taler.net/dashboards">`_ and easily
-add the new dashboard for the Nginx Exporter program. Please make sure you choose the right Prometheus data source.
-
-* Dashboard Id: 11199
-* Dashboard URL: https://grafana.com/grafana/dashboards/11199-nginx/
diff --git a/system-administration/prometheus-alerts.rst b/system-administration/prometheus-alerts.rst
@@ -1,176 +0,0 @@
-..
- This file is part of GNU TALER.
- Copyright (C) 2014-2023 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU Affero General Public License as published by the Free Software
- Foundation; either version 3.0, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
-
- @author Javier Sepulveda
-
-Prometheus alerts
-#################
-
-The Prometheus alert manager does periodically queries defined in the alert rules files.
-In the event that any of these conditions are met, then the alerting system will send a notification (i.e email),
-directly to the specified contact points, or towards a specific group of contact points (these are named notification policies).
-
-Very basic concepts of the Prometheus Alerting system, are explained right below. Please check the `Prometheus official
-documentation <https://prometheus.io/docs/alerting/latest/overview/>`_ , or the `Grafana documentation <https://grafana.com/docs/grafana/latest/alerting/fundamentals/>`_, if you need to get some additional information.
-
-* Alert rules: One or more queries (expresions) to measure (i.e disk space, memory, or cpu usage).
-
- - Each alert rule contains a condition with a specific threshold.
- - Each alert rule can contain a precise contact point to send the notifications to.
- - Within the same alert rule, you can specify multiple alert instances.
-
-* Contact points: This is the message notification itself, in conjunction with the specific address to send the notification to.
-
-* Notification policies: This feature allows you to gather a group different contact points, under the same label name.
-
-Install Prometheus alert manager
-================================
-
-.. code-block:: console
-
- # apt install prometheus-alertmanager
- # systemctl start prometheus-alertmanager
- # systemctl status protheus-alertmanager
-
-Edit the Prometheus configuration file
-======================================
-
-To make Prometheus talk with the alerting system, you need to
-speficy this, on the main prometheus configuration file.
-
-.. code-block:: yaml
-
- # Path: /etc/prometheus/prometheus.yml
- # Add this at the end of yml file
- # Alertmanager configuration
- alerting:
- alertmanagers:
- - static_configs:
- - targets: ["localhost:9093"]
-
-Alert rules configuration file
-==============================
-
-- Create your very first first_rule.yml file.
-
- .. note::
-
- The code shown below, is just an example for CPU, disk and memory usage.
-
-.. code-block:: yaml
-
- # Path: /etc/prometheus/alert_rules.yml
- groups:
- - name: node_exporter_alerts
- rules:
- - alert: HighCPULatency
- expr: sum(rate(node_cpu_seconds_total{mode="system"}[1m])) / count(node_cpu_seconds_total{mode="system"}) * 100 > 80
- for: 1m
- labels:
- severity: warning
- annotations:
- summary: "High CPU Latency detected"
- description: "CPU latency is above 80% for more than 1 minute."
-
- - alert: LowDiskSpace
- expr: (node_filesystem_free_bytes / node_filesystem_size_bytes) * 100 < 10
- for: 1m
- labels:
- severity: critical
- annotations:
- summary: "Low Disk Space detected"
- description: "Disk space is below 10% for more than 1 minute."
-
- - alert: HighMemoryUsage
- expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 80
- for: 1m
- labels:
- severity: warning
- annotations:
- summary: "High Memory Usage detected"
- description: "Memory usage is above 80% for more than 1 minute."
-
-
-Configure SMTP
-==============
-
-.. code-block:: yaml
-
- # Path: /etc/prometheus/alertmanager.yml
-
- global:
- smtp_smarthost: 'smtp.example.com:587'
- smtp_from: 'alertmanager@example.com'
- smtp_auth_username: 'yourusername'
- smtp_auth_password: 'yourpassword'
-
- route:
- receiver: 'email'
-
- receivers:
- - name: 'email'
- email_configs:
- - to: 'recipient@example.com'
- send_resolved: true
-
-
-Add your alert rules to Prometheus
-==================================
-
-.. code-block:: yaml
-
- #Path: /etc/prometheus/prometheus.yml
- # Add here your alert_rules.yml files
- rule_files:
- - "first_rule.yml"
- - # "second_rule.yml"
-
-
-Edit the alertmanager systemd service file
-============================================
-
-.. code-block:: systemd
-
- # Path: /usr/lib/systemd/system/prometheus-alertmanager.service
-
- [Unit]
- Description=Alertmanager for prometheus
- Documentation=https://prometheus.io/docs/alerting/alertmanager/
-
- [Service]
- Restart=on-failure
- User=prometheus
- EnvironmentFile=/etc/default/prometheus-alertmanager
- ExecStart=/usr/bin/prometheus-alertmanager \
- --cluster.advertise-address="ip:9093" # Add this, as otherwise it won't work
- ExecReload=/bin/kill -HUP $MAINPID
- TimeoutStopSec=20s
- SendSIGKILL=no
-
- [Install]
- WantedBy=multi-user.target
-
-
-.. code-block:: console
-
- # systemctl daemon-reload
- # systemctl restart prometheus-alertmanager
- # systemctl restart prometheus
-
-Check
-=====
-
-You can check both your rules (http://ip:9090/rules) and alerts (http://ip:9090/alerts), from your web browser.
-
diff --git a/system-administration/prometheus-node-exporter.rst b/system-administration/prometheus-node-exporter.rst
@@ -1,106 +0,0 @@
-..
- This file is part of GNU TALER.
- Copyright (C) 2014-2023 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU Affero General Public License as published by the Free Software
- Foundation; either version 3.0, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
-
- @author Javier Sepulveda
-
-
-Prometheus node-exporter
-########################
-
-Create a system user
-====================
-
-.. code-block:: console
-
- # useradd --system --no-create-home --shell /bin/false node_exporter
-
-
-Download Prometheus node-exporter
-=================================
-
-* Download
-* Extract
-* Copy to /usr/local/bin
-* Set ownership and permissions
-
-.. code-block:: console
-
- # cd /tmp
- # wget https://github.com/prometheus/node_exporter/releases/download/v1.8.1/node_exporter-1.8.1.linux-amd64.tar.gz
- # mv node_exporter-1.8.1.linux-amd64.tar.gz node_exporter.tar.gz
- # tar -xzvf node_exporter.tar.gz
- # cp node_exporter/node_exporter /usr/local/bin
- # chown node_exporter:node_exporter /usr/local/bin/node_exporter
-
-Modify Prometheus configuration file
-====================================
-
-.. code-block:: yaml
-
- # Path: /etc/prometheus/prometheus.yml
-
- - job_name: 'node_exporter_metrics'
- scrape_interval: 5s
- static_configs:
- - targets: ['localhost:9100']
-
-
-Systemd node-exporter service file
-==================================
-
-.. code-block:: systemd
-
- # Path: /etc/systemd/system/node-exporter.service
-
- [Unit]
- Description=Node Exporter
- After=network.target
-
- [Service]
- User=node_exporter
- Group=node_exporter
- Type=simple
- ExecStart=/usr/local/bin/node_exporter
-
- [Install]
- WantedBy=multi-user.target
-
-
-Refresh systemd and restart Prometheus
---------------------------------------
-
-.. code-block:: console
-
- # systemctl daemon-reload
- # systemctl enable --now node_exporter
- # systemctl status node_exporter
- # systemctl restart prometheus
-
-Check
------
-
-http://ip:9100/
-
-
-Grafana control panel (GUI)
-===========================
-
-You can now go to the `Grafana dashboards <a href="https://grafana.taler.net/dashboards">`_ and easily
-add a new dashboard for the Node Exporter program. Please make sure you choose the right Prometheus data source.
-
-* Dashboard Id: 1860
-* Dashboard URL: https://grafana.com/grafana/dashboards/1860-node-exporter-full/
-
-
diff --git a/system-administration/prometheus-postgres-exporter.rst b/system-administration/prometheus-postgres-exporter.rst
@@ -1,171 +0,0 @@
-..
- This file is part of GNU TALER.
- Copyright (C) 2014-2023 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU Affero General Public License as published by the Free Software
- Foundation; either version 3.0, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
-
- @author Javier Sepulveda
-
-
-Prometheus postgres-exporter
-############################
-
-Create a system user
-====================
-
-.. code-block:: console
-
- # useradd -s /sbin/nologin --system postgres_exporter
- # groupadd --system postgres_exporter
-
-Download Prometheus node-exporter
-=================================
-
-* Download
-* Extract
-* Copy to /usr/local/bin
-* Set ownership and permissions
-
-.. code-block:: console
-
- # cd /tmp
- # wget https://github.com/prometheus-community/postgres_exporter/releases/download/v0.15.0/postgres_exporter-0.15.0.linux-amd64.tar.gz
- # tar -xzvf https://github.com/prometheus-community/postgres_exporter/releases/download/v0.15.0/postgres_exporter-0.15.0.linux-amd64.tar.gz
- # cd postgres_exporter-0.12.0.linux-amd64
- # cp postgres_exporter /usr/local/bin
- # chown postgres_exporter:postgres_exporter /usr/local/bin/postgres_exporter
-
-Create environment variable
-===========================
-
-.. code-block:: console
-
- # mkdir -p /opt/postgres_exporter
- # cd /opt/postgres_exporter
- # touch postgres_exporter.env
- # Paste next content:
- DATA_SOURCE_NAME="postgresql://postgres_exporter:PASSWORD@localhost:5432/?sslmode=disable"
-
-
-Create postgres-exporter.sql file
-=================================
-
-.. code-block:: console
-
- # cd /opt/postgres_exporter
- # touch postgres-exporter.sql
-
-.. code-block::
-
- -- To use IF statements, hence to be able to check if the user exists before
- -- attempting creation, we need to switch to procedural SQL (PL/pgSQL)
- -- instead of standard SQL.
- -- More: https://www.postgresql.org/docs/9.3/plpgsql-overview.html
- -- To preserve compatibility with <9.0, DO blocks are not used; instead,
- -- a function is created and dropped.
- CREATE OR REPLACE FUNCTION __tmp_create_user() returns void as $$
- BEGIN
- IF NOT EXISTS (
- SELECT -- SELECT list can stay empty for this
- FROM pg_catalog.pg_user
- WHERE usename = 'postgres_exporter') THEN
- CREATE USER postgres_exporter;
- END IF;
- END;
- $$ language plpgsql;
-
- SELECT __tmp_create_user();
- DROP FUNCTION __tmp_create_user();
-
- ALTER USER postgres_exporter WITH PASSWORD 'password';
- ALTER USER postgres_exporter SET SEARCH_PATH TO postgres_exporter,pg_catalog;
-
- -- If deploying as non-superuser (for example in AWS RDS), uncomment the GRANT
- -- line below and replace <MASTER_USER> with your root user.
- -- GRANT postgres_exporter TO <MASTER_USER>;
-
- GRANT CONNECT ON DATABASE postgres TO postgres_exporter;
-
-
-Login in Postgres
-=================
-
-Login into Postgres and execute the previous file, as the postgres user.
-
-.. code-block:: console
-
- # su -c "psql" postgres
- # \i postgres-exporter.sql
-
-
-Systemd postgres-exporter service file
-======================================
-
-.. code-block:: systemd
-
- # Path: /etc/systemd/system/postgres-exporter.service
-
- [Unit]
- Description=Prometheus exporter for Postgresql
- Wants=network-online.target
- After=network-online.target
-
- [Service]
- User=postgres_exporter
- Group=postgres_exporter
- WorkingDirectory=/opt/postgres_exporter
- EnvironmentFile=/opt/postgres_exporter/postgres_exporter.env
- ExecStart=/usr/local/bin/postgres_exporter --web.listen-address=:9187 --web.telemetry-path=/metrics
- Restart=always
-
- [Install]
- WantedBy=multi-user.target
-
-
-
-Edit Prometheus configuration file
-===================================
-
-* Add this new job to the /etc/prometheus/prometheus.yml configuration file.
-
-.. code-block:: yaml
-
- - job_name: 'postgres_exporter'
- static_configs:
- - targets: ['localhost:9187']
-
-
-Refresh systemd
-===============
-
-.. code-block:: console
-
- # systemctl daemon-reload
- # systemctl enable --now postgres_exporter.service
- # systemctl status postgres_exporter.service
- # systemctl restart prometheus
-
-Check if new postgres-exporter service, is up and running properly: http://ip:9187/
-
-Grafana control panel (GUI)
-===========================
-
-You can now go to `Grafana dashboards <a href="https://grafana.taler.net/dashboards">`_ and easily
-add a new dashboard for the Postgres Exporter program. Please make sure you choose the right Prometheus data source.
-
-* Dashboard Id: 9628
-* Dashboard URL: https://grafana.com/grafana/dashboards/9628-postgresql-database/
-
-
-More information: https://github.com/prometheus-community/postgres_exporter
-
-
diff --git a/system-administration/prometheus.rst b/system-administration/prometheus.rst
@@ -1,218 +0,0 @@
-..
- This file is part of GNU TALER.
- Copyright (C) 2014-2023 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU Affero General Public License as published by the Free Software
- Foundation; either version 3.0, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
-
- @author Javier Sepulveda
-
-Installation of Prometheus
-##########################
-
-Download Prometheus
-===================
-
-* Download
-* Extract
-* Copy files to /usr/local/bin
-* Set ownership and permissions
-
-.. code-block:: console
-
- $ wget https://github.com/prometheus/prometheus/releases/download/v2.52.0/prometheus-2.52.0.linux-amd64.tar.gz -O /tmp/prometheus.tar.gz
- $ cd /tmp
- $ tar -xvzf prometheus.tar.gz
- $ mkdir /etc/prometheus
- $ mkdir /var/lib/prometheus
-
-.. code-block:: console
-
- $ chown prometheus:prometheus /etc/prometheus
- $ chown prometheus:prometheus /var/lib/prometheus/
- $ cp prometheus promtool /usr/local/bin/
- $ chown prometheus:prometheus /usr/local/bin/prometheus
- $ chown prometheus:prometheus /usr/local/bin/promtool
- $ cp -R consoles console_libraries /etc/prometheus/
- $ chown -R prometheus:prometheus /etc/prometheus/consoles
- $ chown -R prometheus:prometheus /etc/prometheus/console_libraries
- $ chown prometheus:prometheus /etc/prometheus/prometheus.yml
-
-
-Create a system user
-====================
-
-.. code-block:: console
-
- $ useradd --no-create-home --shell /bin/false prometheus
-
-
-Configuring Prometheus
-======================
-
-The main configuration file of Prometheus is in the YAML format, and it is located
-on /etc/prometheus/prometheus.yml. Please be aware YAML, is a very sensitive format,
-where white spaces matter.
-
-.. note::
- If you find that your prometheus service is not working properly,
- please always check the prometheus.yml configuration file, or the systemd prometheus.service file,
- as in most of occasions, these are the main source of errors.
-
-
-Prometheus main configuration file
-----------------------------------
-
-.. code-block:: yaml
-
- # Path: /etc/prometheus/prometheus.yml
-
- global:
- scrape_interval: 15s
- evaluation_interval: 15s
-
- scrape_configs:
- - job_name: prometheus
- static_configs:
- - targets: ['localhost:9090']
-
-Prometheus Systemd service file
-===============================
-
-.. code-block:: systemd
-
- # Path: /etc/systemd/system/prometheus.service
-
- [Unit]
- Description=Prometheus
- Wants=network-online.target
- After=network-online.target
-
- [Service]
- User=prometheus
- Group=prometheus
- Type=simple
- ExecStart=/usr/local/bin/prometheus \
- --config.file /etc/prometheus/prometheus.yml \
- --storage.tsdb.path /var/lib/prometheus/ \
- --web.console.templates=/etc/prometheus/consoles \
- --web.console.libraries=/etc/prometheus/console_libraries
-
- [Install]
- WantedBy=multi-user.target
-
-
-Refresh systemd file and restart Prometheus
--------------------------------------------
-
-.. code-block:: console
-
- # systemctl daemon-reload
- # systemctl start prometheus
- # systemctl status prometheus
- # systemctl enable prometheus.service
-
-
-Check
-------
-
-The prometheus service should be listening on the 9090 port, check this with your Internet browser,
-by typing http://ip:9090.
-
-.. note::
-
- If you find any problem to start the prometheus service, use the "journalctl -u prometheus" command
- to find any errors. Always check twice the prometheus.yml and the prometheus.service files.
-
-
-Close the 9090 port with Nginx
-==============================
-
-.. note::
-
- This is optional, but increases security, by closing your server 9090 listening port.
-
-In order to reverse proxy the Prometheus default port (9090) to loopback with NGINX,
-you need to undertake 3 actions:
-
-- Modify the current systemd service file
-- Create a new Nginx virtualhost file to loopback port 9090 to prometheus
-- Restart the Prometheus systemd service file
-
-
-Modify the Prometheus systemd service file
-------------------------------------------
-
-.. code-block:: systemd
-
- ExecStart=/usr/local/bin/prometheus \
- --config.file=/etc/prometheus/prometheus.yml \
- --storage.tsdb.path=/var/lib/prometheus \
- --web.console.templates=/etc/prometheus/consoles \
- --web.console.libraries=/etc/prometheus/console_libraries \
- --web.listen-address=127.0.0.1:9090 \ # Add this line ****
- --web.external-url=/prometheus/ # Add this line ****
-
-
-Nginx configuration
--------------------
-
-.. code-block:: nginx
-
- # Path: /etc/nginx/sites-available/prometheus.conf
-
- server {
- listen 80;
- listen [::]:80;
- server_name name.domain.tld;
- root /dev/null;
-
- location /prometheus/ {
- proxy_pass http://localhost:9090;
- }
- }
-
-.. code-block:: console
-
- ## Create the Nginx symbolic link to enable the virtualhost file
- # ln -s /etc/nginx/sites-available/prometheus.conf /etc/nginx/sites-enabled/prometheus
- # nginx -t
- # systemctl reload nginx
-
-
-Restart the Prometheus service
-------------------------------
-
-.. code-block:: console
-
- # systemctl daemon-reload
- # systemctl restart prometheus
-
-Check
------
-
-If you have done all steps explained above you won't be able to reach Prometheus
-anymore from your web browser (http://ip:9090).
-To reach the prometheus program from your web browser you will have to access through
-the subdomain or domain name, that you specified on the nginx virtualhost file.
-
-
-Grafana control panel (GUI)
-===========================
-
-To add the server prometheus connector or datasource, to the `Taler grafana server <https://grafana.taler.net>`_
-go to "Connections->Add new connection" and search for "Prometheus" in the search text field. Once you have found and selected the Prometheus connection type, just press the blue button "Add new data source".The main 2 fields required to specify are the name of the connector, and the URL. Then just press "Save and test", and you are done.
-
-.. note::
-
- Please find the main official documentation in `Prometheus official documentation <https://prometheus.io/>`_
-
-
diff --git a/system-administration/taler-monitoring-infrastructure.rst b/system-administration/taler-monitoring-infrastructure.rst
@@ -15,188 +15,95 @@
TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
@author Javier Sepulveda
+
.. _taler-merchant-monitoring:
+.. _taler-monitoring:
-GNU Taler monitoring
+GNU Taler monitoring
####################
-.. image:: images/taler-monitoring-infrastructure.png
-
-In order to check the availability of our server infrastructure, we use the Grafana and Uptime KUMA monitoring programs.
-
-On the one hand Grafana enables us to see *graphically* the server consumption resources, and even alert us of some specific situations.
-On the other hand with a more basic tool such as Uptime KUMA (which does mostly ping and https checks),
-we get the very first status information, as a very first countermeasure.
-
-Grafana
-=======
-
-- Our grafana instance can be reached at https://grafana.taler.net
-- Our grafana instance is installed on the (TUE) server
-
-User accounts:
---------------
-
-We have only two main user accounts:
-
-- One "admin" account for server administrators.
-- One general "read-only" account, for the rest of the team.
-
-How to install Grafana
-----------------------
-
-Please refer to the Grafana official website for installation instructions for your specific operating system. For the
-specific case of the GNU/Linux distribution Debian 13 (trixie), you can use the next set of instructions.
-
-.. code-block:: console
-
- # apt-get install -y apt-transport-https
- # apt-get install -y software-properties-common wget
- # wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key
- # echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" | tee -a /etc/apt/sources.list.d/grafana.list
- # apt update
- # apt-get install grafana
- # systemctl daemon-reload
- # systemctl enable --now grafana-server
-
-.. note::
-
- If you want to deploy grafana automatically, and if you have access to the --private git repository "migration-exercise-stable.git",
- please clone it, and execute from Grafana subfolder the grafana.sh file. This script will install for you Grafana and will leave it up and running on port 3000 of your server.
-
-Grafana Dashboards
-------------------
-
-As we understand creating tailored Grafana dashboards, is very time consuming thing to do, and in the top of that
-you really have to to be very proficient to do that, we use the available and pre-built `Grafana dashboards <https://grafana.com/grafana/dashboards/>`_, which eventually we can also tweak a little, to fit our needs.
-
-Node Exporter
-++++++++++++++
-
-- More information can be found on the `Node Exporter <https://grafana.com/grafana/dashboards/1860-node-exporter-full/>`_ website.
-- Dashboard ID: 1860
-
-.. note::
-
- If you want to deploy Postgres Exporter automatically and have access to the --private git repository "migration-exercise-stable.git", please clone it,
- and execute from the subfolder taler.net/grafana/node-exporter.sh, this script will install for you Node Exporter and will leave it running on port 9100.
- This script also will create, start, and enable on reboot a new service.
-
-Postgres Exporter
-+++++++++++++++++
-
-- More information can be found on the `PostgreSQL exporter <https://grafana.com/grafana/dashboards/12485-postgresql-exporter/>`_ website.
-- Dashboard ID: 12485
-
-.. image:: images/grafana-postgres-exporter.png
-
-.. note::
-
- If you want to deploy Postgres Exporter automatically and have access to the --private git repository "migration-exercise-stable.git", please clone it,
- and execute from the subfolder taler.net/grafana/postgres-exporter.sh, this script will install for you Grafana and will leave it running on port 9187.
-
-Uptime Kuma from Grafana
-++++++++++++++++++++++++
-
-This is an easy to way to integrate all monitored websites from Uptime Kuma, into Grafana. Thus,
-from the same place (Grafana), you can check also the status of the website and the expiration date of the
-certificates.
-
-- More information can be found on the `Uptime Kuma for Grafana <https://grafana.com/grafana/dashboards/18278-uptime-kuma/>`_ website.
-- Dashboard ID: 18278
-
-.. image:: images/uptime-kuma-from-grafana.png
-
-Grafana Data Sources
----------------------
-As a data source connector we use Prometheus.
-
-Prometheus
-++++++++++
-More information can be found in the `Grafana and Prometheus <https://grafana.com/docs/grafana/latest/getting-started/get-started-grafana-prometheus/>`_ website.
-
-.. note::
-
- If you want to deploy Prometheus automatically and have access to the --private git repository "migration-exercise-stable.git", please clone it,
- and execute from the subfolder taler.net/grafana/prometheus.sh, this script will install for you Grafana and will leave it running on port 9090.
-
-Managing logs
--------------
-
-In order to manage logs, we use Loki + Promtail (Debian packages), which are very easy to integrate with Grafana and Prometheus.
-
-.. code-block:: console
-
- # Install
- # apt-get install loki promtail
- # Start services
- # systemctl start loki promtail
- # Enable services on reboot
- # systemctl enable loki
- # systemctl enable promtail
-
-Loki and Promtail services in Grafana
-----------------------------------------------
-
-1) Make sure you have prometheus running on port 9090
-2) Make sure you have loki running on port 3100
-
-.. code-block:: console
-
- systemctl status prometheus loki
-
-
-.. note::
-
- We still don't have Loki and Promtail installed in production (taler.net), and neither
- configured to track certain log files.
-
-Grafana Alerting
-----------------
-
-#. In order to use the Grafana alerting system rules, you need first to configure working SMTP service of your server.
-#. Once you have done the necessary changes on the Grafana configuration file, you have to either restart or reload the "grafana-server" service with the systemctl command as usual.
-#. Then go to the Grafana admin panel Alerting -> Contact points, and within the email address you are using for this purpose, check if SMTP is indeed working by pressing the "test" button.
-#. If that works, you will receive an email in your mailbox with the Grafana logo confirming that the server can satisfactorily send email messages.
-
-
-Uptime Kuma
-===========
-
-- URL: https://uptimekuma.anastasis.lu (main)
-- Users: One single administration account with full privileges.
-- Installation: Without docker. All within the user home folder /home/uptime-kuma
-- Monitors almost all our servers, websites and certificates expiration dates.
-
-- URL: https://uptimekuma.taler.net
-- Users: One single administration account with full privileges.
-- Installation: Without docker. All within the user home folder /home/uptime-kuma
-- Monitors the "main" uptimekuma installation, to make sure it is up and running, and doing the monitoring properly.
-
-.. image:: images/kuma.png
-
-.. note::
-
- 1) The main uptimekuma installation is under the server anastasis.lu
- 2) The second uptimekuma installation on top, is installed on gv.taler.net.
-
-Kuma monitor types
--------------------
-
-Kuma counts with quite a few monitor types, such as https, TCP port or ping. In our case, we use mainly https requests,
-and pings, to make sure as a first check that our servers are responsive.
-
-Another handy feature that Kuma has, is the "Certificate Expiry Notification feature, which we also use, and eventually warn us about a certificate
-expiration dates.
-
-So in brief in our KUMA main server, we use these 3 monitor types (ping,https,certificate expiration) for each website that we monitor.
-
-Exceptionally for high priority notifications for essential services, and specifically due of the importance of the Taler Operations production
-server, we use in addition SMS notifications (Clicksend provider). This way in the case the main uptimekuma detecting the Taler Operations server unavailability, or any other essential service such as GIt,
-a SMS message would be sent to the system administrator and eventually some other team member of the deployment and operations department, for urgent action.
-
-
-How to edit notifications:
-
-.. image:: images/uptime-kuma-edit.png
-
+Sentol is the central monitoring server for host metrics, journals and service
+health. The Ansible setup and operational documentation live in the
+``monitoring/`` directory of ``tsys-infra.git``, starting with
+``monitoring/README.md``. Refer there for deployment, configuration, client
+enrollment and maintenance.
+
+Architecture
+============
+
+The stack runs as native systemd services on Sentol and the monitored hosts:
+
+* **Metrics:** Prometheus scrapes Node Exporter and Fluent Bit metrics through
+ each host's nginx proxy using mutually authenticated TLS (mTLS).
+* **Logs:** Fluent Bit forwards system journals through Sentol's mTLS nginx
+ receiver to VictoriaLogs. The receiver derives the host identity from the
+ client certificate.
+* **Service checks:** Blackbox Exporter probes HTTP(S) availability and TLS
+ certificates; JSON Exporter reads build and protocol versions from service
+ ``/config`` endpoints.
+* **Dashboards and alerts:** Perses displays Prometheus metrics. Prometheus
+ evaluates alert rules and sends alerts to Alertmanager for email delivery.
+
+.. only:: html
+
+ .. mermaid::
+ :config: {"layout": "elk"}
+ :caption: Monitoring architecture; arrows show requests or log and alert delivery.
+ :alt: Prometheus scrapes host metrics over mTLS and service health through exporters. Fluent Bit sends journals over mTLS to VictoriaLogs. Perses provides dashboards, Alertmanager sends email, and nginx provides authenticated web access.
+
+ flowchart TB
+ subgraph host["Monitored host"]
+ proxy["nginx metrics proxy"]
+ node["Node Exporter"]
+ journal["System journal"]
+ fluent["Fluent Bit"]
+ proxy --> node
+ proxy -->|"collector metrics"| fluent
+ journal --> fluent
+ end
+ subgraph sentol["Sentol"]
+ prom["Prometheus"]
+ ingest["nginx log receiver"]
+ logs["VictoriaLogs"]
+ probes["Blackbox and JSON exporters"]
+ perses["Perses"]
+ alert["Alertmanager"]
+ web["nginx web access"]
+ prom -->|"scrape"| probes
+ perses -->|"query"| prom
+ prom -->|"alerts"| alert
+ ingest --> logs
+ web --> perses
+ web --> prom
+ web --> alert
+ web --> logs
+ end
+ prom -->|"pull / mTLS"| proxy
+ fluent -->|"push / mTLS"| ingest
+ probes -->|"HTTP(S) and /config"| endpoints["Public services"]
+ browser["Operator browser"] -->|"HTTPS / password"| web
+ alert --> smtp["SMTP relay"]
+ kuma["Uptime Kuma"] -->|"independent checks"| endpoints
+
+Sentol also monitors itself. Collector and server backends listen on loopback;
+nginx exposes the authenticated interfaces. One private monitoring CA secures
+metrics and log transport. Metrics and logs are stored centrally on Sentol.
+
+Hosts are either managed directly by ``tsys-infra.git`` or enrolled as external
+clients. ``ansible-taler-exchange`` uses the latter model: it configures exchange
+hosts from bundles exported by Sentol.
+
+:doc:`uptime-kuma` provides independent availability monitoring, including
+exchange key and fee checks. Its checks and notifications are configured
+separately from Sentol.
+
+Web interfaces
+==============
+
+The interfaces share password authentication over HTTPS. Public Perses access
+is read-only.
+
+* `Perses <https://monitoring.taler.net/>`_: dashboards and monitoring overview.
+* `Prometheus <https://monitoring.taler.net/prometheus/>`_: metrics, targets and rules.
+* `Alertmanager <https://monitoring.taler.net/alertmanager/>`_: alerts and silences.
+* `VictoriaLogs <https://monitoring.taler.net/logs/>`_: journal search.
diff --git a/system-administration/uptime-kuma.rst b/system-administration/uptime-kuma.rst
@@ -18,6 +18,11 @@
Uptime Kuma
###########
+Uptime Kuma remains in use alongside :ref:`taler-monitoring`. It provides
+independent public availability checks and the exchange key and fee checks
+described below. Its monitors and notifications are configured separately from
+Sentol's Prometheus probes and Alertmanager routing.
+
We use https://github.com/louislam/uptime-kuma to monitor public server and service
availability. We run two servers:
