diff --git a/flake.lock b/flake.lock index 8376716..c4be3be 100644 --- a/flake.lock +++ b/flake.lock @@ -14,11 +14,11 @@ "utils": "utils" }, "locked": { - "lastModified": 1652293408, - "narHash": "sha256-lj38EcFe6Y4m23m9Bmqste6KX7haDFE51u7wlbBeLo4=", + "lastModified": 1652372221, + "narHash": "sha256-C/L/MtUw2ZrBGitAqQdrd8XfOlRMvyQ5pXklMe4kztc=", "owner": "dump-dvb", "repo": "data-accumulator", - "rev": "1e0c67be519c03cc03febff604186e8281ad74e5", + "rev": "1d4796a4bb469248a25c97fa8a3f571da6776fd5", "type": "github" }, "original": { @@ -34,11 +34,11 @@ "utils": "utils_2" }, "locked": { - "lastModified": 1652291160, - "narHash": "sha256-fDIZ7Rtm77J9lB+iY9UXtlGF/iKSUQIEVZ4K1cpvp7g=", + "lastModified": 1652372540, + "narHash": "sha256-mkpXYip4VjB/HEiXmb+R2uWha/5ShywPrxXjzeU6ETY=", "owner": "dump-dvb", "repo": "decode-server", - "rev": "6bcfa32304434473f4c264aed040eff958bd8e05", + "rev": "e28fb00212fff2bccadc7277e74bb2fe36f48eba", "type": "github" }, "original": { @@ -78,11 +78,11 @@ "utils": "utils_3" }, "locked": { - "lastModified": 1652292948, - "narHash": "sha256-6BOUZnTkavF/XR6cyy6462hQiC6qiHMYogEvUelwFUU=", + "lastModified": 1652376095, + "narHash": "sha256-p4yp1a7KviYV95rEoZJ9sS1JJFDtvpVd02Fven0LzjQ=", "owner": "dump-dvb", "repo": "dvb-api", - "rev": "4793ba7d7e7274796d9398fa7ca78e291973034e", + "rev": "d7854178c2aafffe33e158591ae39dec9b30777b", "type": "github" }, "original": { @@ -215,11 +215,11 @@ "stops": { "flake": false, "locked": { - "lastModified": 1652290634, - "narHash": "sha256-C5lyj1M1VYYsRebGu+R/6UIT0M3GEJ4xhueuv2BeJ8A=", + "lastModified": 1652377645, + "narHash": "sha256-YuRMr6zxrK4oXgGWtIk7KLjf4acyo4cOAfXgX98oaPI=", "owner": "dump-dvb", "repo": "stop-names", - "rev": "a617888d7844758301827450363ea9cf24a133dd", + "rev": "256992c561d6aa68e573e1cfa05b0a0d560a31d1", "type": "github" }, "original": { @@ -311,11 +311,11 @@ "utils": "utils_5" }, "locked": { - "lastModified": 1652226534, - "narHash": "sha256-k8FC8+cH6V5HyiKwYgjFW2TAf87FG7JuaTUrUI4cfSA=", + "lastModified": 1652308072, + "narHash": "sha256-3bK0wuFcKgALAQTeFrNEAfrqjJ38TnCpqJvgsnMbPws=", "owner": "dump-dvb", "repo": "windshield", - "rev": "b65d33124cdf5bf8dba4f91a9ba76064f80ad5f4", + "rev": "7aff08e69faf8cba1f3f3d2ba7d4a1496548e1b8", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 5a913b0..dfb64ff 100644 --- a/flake.nix +++ b/flake.nix @@ -122,6 +122,7 @@ ./modules/map.nix ./modules/file_sharing.nix ./modules/numbering.nix + ./modules/grafana.nix { nixpkgs.overlays = [ data-accumulator.overlay."x86_64-linux" @@ -130,6 +131,7 @@ docs.overlay."x86_64-linux" ]; dvb-dump.stopsJson = "${stops}/stops.json"; + dvb-dump.graphJson = "${stops}/graph.json"; } ]; }; diff --git a/modules/data-accumulator.nix b/modules/data-accumulator.nix index cbdf962..5620a9d 100644 --- a/modules/data-accumulator.nix +++ b/modules/data-accumulator.nix @@ -29,9 +29,6 @@ services = { influxdb = { enable = true; - settings = { - - }; }; }; diff --git a/modules/grafana.nix b/modules/grafana.nix new file mode 100644 index 0000000..e1abd08 --- /dev/null +++ b/modules/grafana.nix @@ -0,0 +1,112 @@ +{ pkgs, config, lib, ... }: { + + services = { + # metrics collector + prometheus = { + enable = true; + port = 9501; + retentionTime = "2d"; + exporters = { + # node exports hardware information like memory, cpu usage and the like + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9502; + }; + + # used for pinging services and checking their healthiness + blackbox = { + enable = true; + configFile = ../services/blackbox.yaml; + }; + }; + + scrapeConfigs = [ + # hardware information + { + job_name = "data-hoarder hardware"; + static_configs = [{ + targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; + }]; + } + + # checks if the data-accumulator server is running + { + job_name = "blackbox-data-accumulator"; + metrics_path = "/probe"; + params = { module = [ "http_2xx" ]; }; + static_configs = [{ + targets = [ + "10.13.37.1:8080" + ]; + }]; + relabel_configs = [ + { source_labels = [ "__address__" ]; target_label = "__param_target"; } + { source_labels = [ "__param_target" ]; target_label = "instance"; } + { target_label = "__address__"; replacement = "127.0.0.1:9115"; } + ]; + } + + # checks if the dvb-api server is running + { + job_name = "blackbox-dvb-api"; + metrics_path = "/probe"; + params = { module = [ "http_2xx" ]; }; + static_configs = [{ + targets = [ + "api.dvb.solutions" + ]; + }]; + relabel_configs = [ + { source_labels = [ "__address__" ]; target_label = "__param_target"; } + { source_labels = [ "__param_target" ]; target_label = "instance"; } + { target_label = "__address__"; replacement = "127.0.0.1:9115"; } + ]; + } + + ]; + }; + promtail = { + enable = true; + # doesn't have a configFile option so this has to do + configuration = builtins.fromJSON (lib.readFile ../services/promtail.json); + }; + + + # exports systemd logs and other + loki = { + enable = true; + configFile = ../services/loki.yaml; + }; + + # visualizer + grafana = { + enable = true; + domain = "monitoring.dvb.solutions"; + port = 2342; + addr = "127.0.0.1"; + + provision = { + enable = true; + dashboards = [ + { options.path = "${../services/dashboards}"; } + ]; + }; + }; + + # reverse proxy for grafana + nginx = { + enable = true; + virtualHosts = { + "${toString config.services.grafana.domain}" = { + enableACME = true; + forceSSL = true; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}"; + proxyWebsockets = true; + }; + }; + }; + }; + }; +} diff --git a/modules/numbering.nix b/modules/numbering.nix index 814ebb5..ef5496c 100644 --- a/modules/numbering.nix +++ b/modules/numbering.nix @@ -10,6 +10,12 @@ with lib; { default = ../configs/stops.json; description = "stops conig json"; }; + options.dvb-dump.graphJson = mkOption { + type = types.path; + default = ../configs/graph.json; + description = "graph json containing the network graphs"; + }; + } diff --git a/modules/public_api.nix b/modules/public_api.nix index 444fbb3..636cb3e 100644 --- a/modules/public_api.nix +++ b/modules/public_api.nix @@ -10,6 +10,7 @@ environment = { "GRPC_HOST" = "127.0.0.1:50051"; "DEFAULT_WEBSOCKET_HOST" = "127.0.0.1:9001"; + "GRAPH_FILE" = "${config.dvb-dump.graphJson}"; "STOPS_FILE" = "${config.dvb-dump.stopsJson}"; }; diff --git a/services/blackbox.yaml b/services/blackbox.yaml new file mode 100644 index 0000000..4e6e678 --- /dev/null +++ b/services/blackbox.yaml @@ -0,0 +1,10 @@ +modules: + http_2xx: + prober: http + http_post_2xx: + prober: http + http: + method: POST + + http_data_accumulator: + prober: http diff --git a/services/dashboards/hardware.json b/services/dashboards/hardware.json new file mode 100644 index 0000000..1026477 --- /dev/null +++ b/services/dashboards/hardware.json @@ -0,0 +1,333 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Memory", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_memory_MemFree_bytes", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Disk usage" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "(sum by (cpu)(rate(node_cpu_seconds_total{mode!=\"idle\"}[5m])))*100", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "GByte", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "node_filesystem_avail_bytes{device=\"/dev/disk/by-uuid/6d8f4b96-4ccb-4a94-a9b1-bab4a28fc940\", fstype=\"ext4\", instance=\"127.0.0.1:9002\", job=\"espresso.ascii.coffee hardware\", mountpoint=\"/nix/store\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_filesystem_avail_bytes", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Disk usage" + } + ], + "title": "Disk", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Hardware", + "uid": "ioi9sts7z", + "version": 4, + "weekStart": "" +} diff --git a/services/dashboards/logs.json b/services/dashboards/logs.json new file mode 100644 index 0000000..35bc3be --- /dev/null +++ b/services/dashboards/logs.json @@ -0,0 +1,82 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "aq1kAhynk" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "aq1kAhynk" + }, + "expr": "{job=\"systemd-journal\"}", + "refId": "A" + } + ], + "title": "Systemd Log", + "transformations": [], + "type": "logs" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Logs", + "uid": "mB6K0hs7k", + "version": 5, + "weekStart": "" +} diff --git a/services/dashboards/overview.json b/services/dashboards/overview.json new file mode 100644 index 0000000..7f56be0 --- /dev/null +++ b/services/dashboards/overview.json @@ -0,0 +1,972 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "Prometheus", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Overview", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "hidden", + "axisSoftMin": -1, + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": "" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "auto" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "sum(node_systemd_unit_state{state=\"active\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "sum(node_systemd_unit_state{state=\"failed\"})", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "gt", + "value": 0, + "visible": true + } + ], + "title": "Systemd Units", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "aq1kAhynk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "aq1kAhynk" + }, + "expr": "rate({job=\"systemd-journal\"}[1m])", + "refId": "A" + } + ], + "title": "Log Rate", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "semi-dark-red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 20, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "probe_success{job=\"blackbox-ascii-pay\"}", + "interval": "", + "legendFormat": "ascii-pay", + "refId": "A" + } + ], + "title": "ASCII-PAY Status", + "type": "state-timeline" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "probe_duration_seconds{job=\"blackbox-ascii-pay\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "ASCII-PAY Response Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "aq1kAhynk" + }, + "gridPos": { + "h": 29, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 14, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "aq1kAhynk" + }, + "expr": "{job=\"systemd-journal\"}", + "refId": "A" + } + ], + "title": "Logs", + "type": "logs" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1000000000 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "min", + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_memory_MemAvailable_bytes", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_memory_MemTotal_bytes", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "MemAvailable", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": {}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_load1", + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_load5", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "CPU Load", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_filefd_allocated", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Filedescriptors allocated", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 10000000000 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_filesystem_size_bytes{mountpoint=\"/\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "lt", + "value": 10000000000, + "visible": true + } + ], + "title": "Diskspace Available", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 5000000000000 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_network_receive_bytes_total{device=\"ens3\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_l3OBty7z" + }, + "exemplar": true, + "expr": "node_network_transmit_bytes_total{device=\"ens3\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "gt", + "value": 5000000000000, + "visible": true + } + ], + "title": "Network Rx/Tx", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "3b_FO0snk", + "version": 22, + "weekStart": "" +} diff --git a/services/loki.yaml b/services/loki.yaml new file mode 100644 index 0000000..bbd9791 --- /dev/null +++ b/services/loki.yaml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + + diff --git a/services/promtail.json b/services/promtail.json new file mode 100644 index 0000000..0d27158 --- /dev/null +++ b/services/promtail.json @@ -0,0 +1,34 @@ +{ + "server": { + "http_listen_port": 28183, + "grpc_listen_port": 0 + }, + "positions": { + "filename": "/tmp/positions.yaml" + }, + "clients": [ + { + "url": "http://127.0.0.1:3100/loki/api/v1/push" + } + ], + "scrape_configs": [ + { + "job_name": "journal", + "journal": { + "max_age": "12h", + "labels": { + "job": "systemd-journal", + "host": "espresso" + } + }, + "relabel_configs": [ + { + "source_labels": [ + "__journal__systemd_unit" + ], + "target_label": "unit" + } + ] + } + ] +} diff --git a/services/promtail.yaml b/services/promtail.yaml new file mode 100644 index 0000000..5ec3b76 --- /dev/null +++ b/services/promtail.yaml @@ -0,0 +1,21 @@ + +server: + http_listen_port: 28183 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://127.0.0.1:3100/loki/api/v1/push + +scrape_configs: + - job_name: journal + journal: + max_age: 12h + labels: + job: systemd-journal + host: espresso + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + target_label: 'unit'