ansible-taler-exchange

config.alloy (10771B)

---

 // For a full configuration reference, see https://grafana.com/docs/alloy
 logging {
   level = "warn"
 }
 
 // Push the logs to loki
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.write "grafana_loki" {
     endpoint {
         url = "https://loki.taler-systems.com/loki/api/v1/push"
         tenant_id = "{{ TARGET_HOST_NAME }}"
         authorization {
           type = "Bearer"
           credentials = "{{ LOKI_ACCESS_TOKEN }}"
         }
     }
 }
 
 
 // Which log files to monitor: all regular log files with errors
 local.file_match "local_files" {
    path_targets = [
         {
                 "__path__" = "/var/log/*.log",
                 "job"      = "system logs",
                 "hostname" = "{{ TARGET_HOST_NAME }}",
         },
    ]
    sync_period = "5s"
 }
 
 
 // Connect local_files as source to filter_generic_logs
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.source.file "log_scrape" {
     targets    = local.file_match.local_files.targets
     forward_to = [loki.process.filter_generic_logs.receiver]
     tail_from_end = true
 }
 
 // Which log files to monitor: all postgres logs
 local.file_match "postgres_log_files" {
    path_targets = [
         {
                 "__path__" = "/var/log/postgresql/*.log",
                 "job"      = "postgres logs",
                 "hostname" = "{{ TARGET_HOST_NAME }}",
         },
    ]
    sync_period = "5s"
 }
 
 
 // Connect postgres_log_files as source to filter_generic_logs
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.source.file "postgres_log_scrape" {
     targets    = local.file_match.postgres_log_files.targets
     forward_to = [loki.process.filter_generic_logs.receiver]
     tail_from_end = true
 }
 
 // Which log files to monitor: all regular log files with errors
 local.file_match "nginx_errors" {
    path_targets = [
         {
                 "__path__" = "/var/log/nginx/*.err",
                 "job"      = "nginx errors",
                 "hostname" = "{{ TARGET_HOST_NAME }}",
         },
    ]
    sync_period = "5s"
 }
 
 // Connect nginx_errors directly to loki
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.source.file "nginx_error_scrape" {
     targets    = local.file_match.nginx_errors.targets
     forward_to = [loki.write.grafana_loki.receiver]
     tail_from_end = true
 }
 
 
 // Which log files to monitor: nginx regular logs
 local.file_match "http_logs" {
    path_targets = [
         {
                 "__path__" = "/var/log/nginx/*.log",
                 "job"      = "nginx logs",
                 "hostname" = "{{ TARGET_HOST_NAME }}",
         },
    ]
    sync_period = "5s"
 }
 
 // Connect http_files as source to filter_http
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.source.file "http_scrape" {
     targets    = local.file_match.http_logs.targets
     forward_to = [loki.process.filter_http.receiver]
     tail_from_end = true
 }
 
 // Filter the HTTP logs
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.process "filter_http" {
 
      // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#stageregex-block
     stage.regex {
       expression  = "(?P<ip>\\S+) (?P<identd>\\S+) (?P<user>\\S+) \\[(?P<timestamp>[\\w:\\/]+\\s[+\\\\-]\\d{4})\\] \"(?P<action>\\S+)\\s?(?P<path>\\S+)\\s?(?P<protocol>\\S+)?\" (?P<status>\\d{3}|-) (?P<size>\\d+|-)\\s?\"?(?P<referrer>[^\\\"]*)\"?\\s?\"?(?P<useragent>[^\\\"]*)?\"?"
     }
 
     // exported via http://localhost:12345/metrics to Prometheus
     stage.metrics {
       metric.histogram {
         name = "http_status_codes"
         prefix = "taler_requests_"
         description = "HTTP status codes, reported from Nginx (all requests)"
         source = "status"
         max_idle_duration = "24h"
         buckets = [100,199,200,201,202,203,299,300,399,400,401,402,403,404,405,406,407,408,409,410,411,418,419,420,450,451,452,499,500,599]
       }
 
       // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metriccounter-block
       metric.counter {
         name = "total_requests"
         prefix = "taler_requests_"
         description = "Total Requests"
         match_all = true
         action = "inc"
       }
     }
 
     stage.drop {
         source = "http_logs"
         expression  = ".*GET.* 200 .*"
         drop_counter_reason = "successful HTTP GETs"
       }
     forward_to = [loki.write.grafana_loki.receiver]
 }
 
 
 // Monitor the logs with the latency statistics
 local.file_match "nginx_taler_performance_logs" {
   path_targets    = [
      {
          "__path__"  = "/var/log/nginx/*.tal",
          "job"       = "nginx/performance",
          "hostname"  = "{{ TARGET_HOST_NAME }}",
      },
   ]
   sync_period = "5s"
 }
 
 
 // Connect nginx_taler_performance_logs as source to perf_logs
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.source.file "perf_scrape" {
    targets    = local.file_match.nginx_taler_performance_logs.targets
    forward_to = [loki.process.perf_logs.receiver]
    tail_from_end = true
 }
 
 
 
 // Here we export the *.tal logs with the Nginx latency data.
 // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/
 loki.process "perf_logs" {
 
   // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#stageregex-block
   stage.regex {
     expression  = "uri=/(?P<ep>[a-zA-Z]+)(?:/\\w+)?(?:/(?P<act>[a-zA-Z-]+))? s=(?P<status>\\d{3}).*urt=(?P<urt>\\d+\\.\\d+|-) rt=(?P<response_time>\\d+\\.\\d+) rl=(?P<request_length>\\d+) bs=(?P<bytes_sent>\\d+)"
   }
 
   // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#stagetemplate-block
   stage.template {
     source = "endpoint"
     template = "{{ '{{' }} printf \"%s-%s\" .ep .act | trimSuffix \"-\" {{ '}}' }}"
   }
 
   stage.template {
     source = "upstream_response_time"
     template = "{{ '{{' }} .urt | replace \"-\" \"0\" {{ '}}' }}"
   }
 
     // exported via http://localhost:12345/metrics to Prometheus
   stage.metrics {
     // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metriccounter-block
     metric.gauge {
        name = "response_time"
        prefix = "taler_requests_"
        description = "Time taken for Nginx to respond (non-GET requests)"
        source = "response_time"
        max_idle_duration = "24h"
        action = "set"
     }
     // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metrichistogram-block
     // https://www.robustperception.io/how-does-a-prometheus-histogram-work
     metric.histogram {
       name = "request_length_hist"
       prefix = "taler_requests_"
       description = "Request Length reported from Nginx (non-GET requests)"
       source = "request_length"
       max_idle_duration = "24h"
       buckets = [1,10,50,100,200,500,1000,2000,5000]
     }
 
     metric.histogram {
       name = "bytes_sent_hist"
       prefix = "taler_requests_"
       description = "Number of bytes sent, reported from Nginx (non-GET requests)"
       source = "bytes_sent"
       max_idle_duration = "24h"
       buckets = [1,10,50,100,200,500,1000,2000,5000]
     }
     metric.histogram {
       name = "response_time_hist"
       prefix = "taler_requests_"
       description = "Time taken for Nginx to respond (non-GET requests)"
       source = "response_time"
       max_idle_duration = "24h"
       buckets = [0.001,0.0025,0.005,0.010,0.025,0.050,0.1,0.25,0.5,1,2,5]
     }
     metric.histogram {
       name = "upstream_response_time_hist"
       prefix = "taler_requests_"
       description = "Time taken for the Exchange to respond to Nginx (non-GET requests)"
       source = "upstream_response_time"
       max_idle_duration = "24h"
       buckets = [0.001,0.0025,0.005,0.010,0.025,0.050,0.1,0.25,0.5,1,2,5]
     }
 
   }
   // Finally, pass on to Loki
   forward_to  = [loki.write.grafana_loki.receiver]
 
 }
 
 
 // Monitor journald logs
 // Export journald logs to our generic filter
 // but first pass to our generic filter to change labels
 loki.source.journal "read"  {
   forward_to    = [loki.process.filter_generic_logs.receiver]
   relabel_rules = loki.relabel.journal.rules
   max_age       = "12h"
   labels        = {component = "loki.source.journal"}
 }
 
 
 // https://community.grafana.com/t/scrape-journald-log-with-alloy-docker-container/119896
 loki.relabel "journal" {
   forward_to = []
   rule {
     source_labels = ["__journal__systemd_unit"]
     target_label  = "systemd_unit"
   }
   rule {
     source_labels = ["__journal__hostname"]
     target_label = "systemd_hostname"
   }
   rule {
     source_labels = ["__journal__transport"]
     target_label = "systemd_transport"
   }
 }
 
 
 // Generic filter for logs
 // See: https://grafana.com/docs/alloy/latest/tutorials/send-logs-to-loki/
 loki.process "filter_generic_logs" {
     // Determine log level:
     // https://community.grafana.com/t/extract-log-level-via-regex-and-set-it-as-a-label/134938/5
     stage.regex {
       expression = `(?P<level>(?i)\b(info|debug|error|warn|warning|trace|fatal)\b)`
     }
 
     // https://grafana.com/docs/alloy/latest/tutorials/processing-logs/
     // Drop debug
     stage.drop {
         source = "level"
         value = "debug"
         drop_counter_reason = "boring debugging data"
     }
     // Drop info
     stage.drop {
         source = "level"
         value = "info"
         drop_counter_reason = "boring info logs"
     }
     // Drop trace
     stage.drop {
         source = "level"
         value = "trace"
         drop_counter_reason = "boring info logs"
     }
 
     stage.metrics {
       // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metriccounter-block
 
       // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#metrichistogram-block
       // https://www.robustperception.io/how-does-a-prometheus-histogram-work
       metric.counter {
         name = "warn_log_level"
         prefix = "system_logs_"
         description = "Warnings in system logs"
         source = "level"
         value = "warn"
         action = "inc"
       }
       metric.counter {
         name = "warn_log_level"
         prefix = "system_logs_"
         description = "Warnings in system logs"
         source = "level"
         value = "warning"
         action = "inc"
       }
       metric.counter {
         name = "error_log_level"
         prefix = "system_logs_"
         description = "Errors in system logs"
         source = "level"
         value = "error"
         action = "inc"
       }
       metric.counter {
         name = "fatal_log_level"
         prefix = "system_logs_"
         description = "Fatal reports in system logs"
         source = "level"
         value = "fatal"
         action = "inc"
       }
     }
 
     forward_to = [loki.write.grafana_loki.receiver]
 }