aboutsummaryrefslogtreecommitdiff
path: root/hosts/sagittarius/web/observability.nix
blob: 1d9277285dcd26285aa731a30cc69d2d1f27d203 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
{ config, ... }:
let
  otel-prometheus-exporter-port = 4316;

  grafana-cfg = config.services.grafana;
  prometheus-cfg = config.services.prometheus;
in
{
  services.opentelemetry-collector = {
    enable = true;
    settings = {
      receivers = {
        otlp = {
          protocols = {
            grpc.endpoint = "localhost:4317";
            http.endpoint = "localhost:4318";
          };
        };
        hostmetrics.scrapers = {
          cpu = { };
          disk = { };
          filesystem = { };
          load = { };
          memory = { };
          network = { };
          process = { };
          processes = { };
          paging = { };
        };
      };
      processors = {
        memory_limiter = {
          check_interval = "5s";
          limit_mib = 2500;
          spike_limit_mib = 500;
        };
        batch = {
          timeout = "10s";
          send_batch_size = 100;
        };
      };
      exporters = {
        debug = {
          verbosity = "detailed";
        };
        prometheus = {
          endpoint = "localhost:${toString otel-prometheus-exporter-port}";
          send_timestamps = true;
          enable_open_metrics = true;
        };
      };
      service = {
        extensions = [ ];
        pipelines = {
          traces = {
            receivers = [ "otlp" ];
            exporters = [ "debug" ];
          };
          metrics = {
            receivers = [
              "otlp"
              "hostmetrics"
            ];
            exporters = [
              "debug"
              "prometheus"
            ];
          };
          logs = {
            receivers = [ "otlp" ];
            exporters = [
              "debug"
            ];
          };
        };
      };
    };
  };

  services.prometheus = {
    enable = true;
    globalConfig = {
      scrape_timeout = "10s";
      scrape_interval = "1m";
    };
    scrapeConfigs = [
      {
        job_name = "otel";
        scrape_interval = "5s";
        static_configs = [
          {
            targets = [ "localhost:${toString otel-prometheus-exporter-port}" ];
          }
        ];
      }
    ];
  };

  services.grafana = {
    enable = true;
    settings = {
      server = {
        http_addr = "127.0.0.1";
        http_port = 9001;
        domain = "sorre.xyz";
        root_url = "https://sorre.xyz/grafana";
        serve_from_sub_path = true;
      };
    };
    provision = {
      enable = true;
      datasources.settings = {
        prune = true;
        datasources = [
          {
            type = "prometheus";
            name = "prometheus";
            url = "http://localhost:${toString prometheus-cfg.port}";
          }
        ];
      };
    };
  };

  services.clickhouse = {
    enable = true;
    usersConfig = {
      otel = {
        profile = "analytics";
        password_sha256_hex = "bf07f7edecb3a2ee3e91f468964c1ef0e03d724d87af08fe8cec0552887b5b68";
        access_management = false;
        default_database = "otel";
      };
    };
    serverConfig = {
      http_port = 8123;
      tcp_port = 9000;
    };
  };

  services.nginx.virtualHosts."grafana.sorre.xyz" = {
    forceSSL = true;
    enableACME = true;
    locations."/" =
      let
        proxyPass =
          "http://"
          + (toString grafana-cfg.settings.server.http_addr)
          + ":"
          + (toString grafana-cfg.settings.server.http_port);
      in
      {
        inherit proxyPass;
        proxyWebsockets = true;
        recommendedProxySettings = true;
      };
  };
}