axion1337.chat-gitops/apps/monitoring/alloy-config.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: alloy-config
  namespace: monitoring
data:
  config.alloy: |
    // Kubernetes pod discovery
    discovery.kubernetes "k8s_pods" {
      role = "pod"
    }

    // Relabel for Prometheus scrape
    discovery.relabel "prometheus_pods" {
      targets = discovery.kubernetes.k8s_pods.targets
      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
        action = "keep"
        regex = "true"
      }
      rule {
        source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
        action = "replace"
        target_label = "__metrics_path__"
        regex = "(.+)"
      }
      rule {
        source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"]
        action = "replace"
        regex = "([^:]+)(?::\\d+)?;(\\d+)"
        replacement = "$1:$2"
        target_label = "__address__"
      }
      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        action = "replace"
        target_label = "namespace"
      }
      rule {
        source_labels = ["__meta_kubernetes_pod_name"]
        action = "replace"
        target_label = "pod"
      }
    }

    // Scrape Flux controllers (flux-system namespace, port 8080)
    discovery.kubernetes "flux_pods" {
      role = "pod"
      namespaces {
        names = ["flux-system"]
      }
    }

    discovery.relabel "flux_scrape" {
      targets = discovery.kubernetes.flux_pods.targets
      rule {
        source_labels = ["__meta_kubernetes_pod_container_port_number"]
        action = "keep"
        regex = "8080"
      }
      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        action = "replace"
        target_label = "namespace"
      }
      rule {
        source_labels = ["__meta_kubernetes_pod_name"]
        action = "replace"
        target_label = "pod"
      }
    }

    // Scrape kube-state-metrics
    prometheus.scrape "kube_state_metrics" {
      targets = [{
        __address__ = "kube-state-metrics.monitoring.svc.cluster.local:8080",
      }]
      forward_to = [prometheus.remote_write.selendis.receiver]
      scrape_interval = "30s"
      scrape_timeout = "10s"
    }

    // Scrape Flux controllers
    prometheus.scrape "flux" {
      targets = discovery.relabel.flux_scrape.output
      forward_to = [prometheus.remote_write.selendis.receiver]
      scrape_interval = "30s"
      scrape_timeout = "10s"
      job_name = "flux"
    }

    // Scrape node-exporter DaemonSet
    prometheus.scrape "node_exporter" {
      targets = [{
        __address__ = "prometheus-node-exporter.monitoring.svc.cluster.local:9100",
      }]
      forward_to = [prometheus.remote_write.selendis.receiver]
      scrape_interval = "30s"
      scrape_timeout = "10s"
    }

    // Scrape Synapse metrics
    prometheus.scrape "synapse" {
      targets = [{
        __address__ = "matrix-stack-synapse-main.matrix.svc.cluster.local:9000",
      }]
      forward_to = [prometheus.remote_write.selendis.receiver]
      scrape_interval = "30s"
      scrape_timeout = "10s"
    }

    // Kubernetes pod logs to Loki
    loki.source.kubernetes "k8s_logs" {
      targets = discovery.kubernetes.k8s_pods.targets
      forward_to = [loki.write.selendis.receiver]
    }

    // Remote write to Selendis Prometheus
    prometheus.remote_write "selendis" {
      endpoint {
        url = "http://10.0.0.3:9090/api/v1/write"
        write_relabel_config {
          source_labels = ["__name__"]
          regex = "go_.*|process_.*"
          action = "drop"
        }
      }
    }

    // Remote write logs to Selendis Loki
    loki.write "selendis" {
      endpoint {
        url = "http://10.0.0.3:3100/loki/api/v1/push"
      }
    }