{ config, ... }:
let
  otel-prometheus-exporter-port = 4316;

  grafana-cfg = config.services.grafana;
  prometheus-cfg = config.services.prometheus;
  # clickhouse-cfg = config.services.clickhouse;
in
{
  services.opentelemetry-collector = {
    enable = true;
    settings = {
      receivers = {
        otlp = {
          protocols = {
            grpc.endpoint = "localhost:4317";
            http.endpoint = "localhost:4318";
          };
        };
        hostmetrics.scrapers = {
          cpu = { };
          disk = { };
          filesystem = { };
          load = { };
          memory = { };
          network = { };
          process = { };
          processes = { };
          paging = { };
        };
      };
      processors = {
        memory_limiter = {
          check_interval = "5s";
          limit_mib = 2500;
          spike_limit_mib = 500;
        };
        batch = {
          timeout = "10s";
          send_batch_size = 100;
        };
      };
      exporters = {
        debug = {
          verbosity = "detailed";
        };
        prometheus = {
          endpoint = "localhost:${toString otel-prometheus-exporter-port}";
          send_timestamps = true;
          enable_open_metrics = true;
        };
        # clickhouse = {
        #   endpoint = "tcp://localhost:${toString clickhouse-cfg.serverConfig.tcp_port}";
        #   database = "otel";
        #   username = "otel";
        #   password = "";
        #   async_insert = true;
        #   ttl = "72h";
        #   compress = "lz4";
        #   create_schema = true;
        #   logs_table_name = "otel_logs";
        #   traces_table_name = "otel_traces";
        #   timeout = "5s";
        #   metrics_tables = {
        #     gauge.name = "otel_metrics_gauge";
        #     sum.name = "otel_metrics_sum";
        #     summary.name = "otel_metrics_summary";
        #     histogram.name = "otel_metrics_histogram";
        #     exponential_histogram.name = "otel_metrics_exp_histogram";
        #   };
        #   retry_on_failure = {
        #     enabled = true;
        #     initial_interval = "5s";
        #     max_interval = "30s";
        #     max_elapsed_time = "300s";
        #   };
        # };
      };
      service = {
        extensions = [ ];
        pipelines = {
          traces = {
            receivers = [ "otlp" ];
            exporters = [
              "debug"
              # "clickhouse"
            ];
          };
          metrics = {
            receivers = [
              "otlp"
              "hostmetrics"
            ];
            exporters = [
              "debug"
              "prometheus"
              # "clickhouse"
            ];
          };
          logs = {
            receivers = [ "otlp" ];
            exporters = [
              "debug"
              # "clickhouse"
            ];
          };
        };
      };
    };
  };

  services.prometheus = {
    enable = true;
    globalConfig = {
      scrape_timeout = "10s";
      scrape_interval = "1m";
    };
    scrapeConfigs = [
      {
        job_name = "otel";
        scrape_interval = "5s";
        static_configs = [
          {
            targets = [ "localhost:${toString otel-prometheus-exporter-port}" ];
          }
        ];
      }
    ];
  };

  services.grafana = {
    enable = true;
    settings = {
      server = {
        http_addr = "127.0.0.1";
        http_port = 9001;
        domain = "grafana.sorre.xyz";
        root_url = "https://grafana.sorre.xyz/";
        serve_from_sub_path = true;
      };
    };
    provision = {
      enable = true;
      datasources.settings = {
        prune = true;
        datasources = [
          {
            type = "prometheus";
            name = "prometheus";
            url = "http://localhost:${toString prometheus-cfg.port}";
          }
        ];
      };
    };
  };

  # services.clickhouse = {
  #   enable = true;
  #   # NOTE: manually created the "otel" user and database
  #   serverConfig = {
  #     http_port = 8123;
  #     tcp_port = 9000;
  #   };
  # };

  services.nginx.virtualHosts."grafana.sorre.xyz" = {
    forceSSL = true;
    enableACME = true;
    locations."/" =
      let
        proxyPass =
          "http://"
          + (toString grafana-cfg.settings.server.http_addr)
          + ":"
          + (toString grafana-cfg.settings.server.http_port);
      in
      {
        inherit proxyPass;
        proxyWebsockets = true;
        recommendedProxySettings = true;
      };
  };
}