using System.Text; using EntKube.Clusters.Domain; using k8s; using k8s.Models; using Microsoft.Extensions.Logging; namespace EntKube.Clusters.Features.DiscoverPrometheus; /// /// Scans a Kubernetes cluster for actual running Prometheus instances. /// We cannot rely on service names or labels alone — anyone can name a service /// "prometheus" without it being one. Instead, this scanner: /// /// 1. Connects to the cluster using the stored kubeconfig /// 2. Lists ALL services across all namespaces /// 3. For each service with an HTTP port, proxies through the Kubernetes API /// to call the service and verify it responds as Prometheus /// 4. Confirms the service is Prometheus by calling /-/ready or /api/v1/query /// /// This approach finds Prometheus regardless of how it was deployed (Helm, /// operator, manual manifests, custom names) because we verify the actual /// running software, not metadata about it. /// public class KubernetesPrometheusScanner : IPrometheusScanner { private readonly ILogger logger; // Per-service probe timeout — don't wait too long on unresponsive services. private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5); public KubernetesPrometheusScanner(ILogger logger) { this.logger = logger; } /// /// Scans all namespaces by listing every service in the cluster, then /// probing each one through the Kubernetes API proxy to confirm whether /// it's actually a Prometheus instance. Returns all confirmed endpoints. /// public async Task> ScanAsync(KubernetesCluster cluster, CancellationToken ct = default) { // Build a Kubernetes client from the cluster's stored kubeconfig. // We use the specific context that was selected during registration. Kubernetes client = BuildClient(cluster); List confirmedEndpoints = new(); // Get the base URI so we can construct absolute proxy URLs. // Using absolute URIs avoids subtle bugs with relative URI resolution // where BaseAddress trailing-slash presence changes the behavior. Uri baseUri = client.BaseUri; logger.LogInformation("Connecting to cluster {ClusterName} at {BaseUri}", cluster.Name, baseUri); // List all services across all namespaces. We don't filter by labels // because we want to find Prometheus instances regardless of how they // were deployed or labelled. V1ServiceList serviceList; try { serviceList = await client.CoreV1.ListServiceForAllNamespacesAsync(cancellationToken: ct); } catch (Exception ex) { logger.LogWarning(ex, "Failed to list services in cluster {ClusterName}. Is the cluster reachable?", cluster.Name); return confirmedEndpoints; } logger.LogInformation("Scanning {ServiceCount} services in cluster {ClusterName} for Prometheus instances", serviceList.Items.Count, cluster.Name); // For each service, check if any of its ports could be Prometheus. // We focus on HTTP-like ports (named http, web, or common Prometheus ports). foreach (V1Service service in serviceList.Items) { if (ct.IsCancellationRequested) { break; } string ns = service.Metadata.NamespaceProperty ?? "default"; string serviceName = service.Metadata.Name; // Skip the kubernetes API service itself. if (serviceName == "kubernetes" && ns == "default") { continue; } List candidatePorts = GetCandidatePorts(service); if (candidatePorts.Count == 0) { continue; } logger.LogInformation("Probing {Service} in {Namespace} on ports [{Ports}]", serviceName, ns, string.Join(", ", candidatePorts)); foreach (int port in candidatePorts) { if (ct.IsCancellationRequested) { break; } // Try to proxy through the Kubernetes API to reach this service // and verify it's Prometheus by calling its endpoints. bool isPrometheus = await ProbeServiceForPrometheusAsync(client, baseUri, ns, serviceName, port, ct); if (isPrometheus) { // Build the in-cluster URL that other services can use to reach this Prometheus. string prometheusUrl = $"http://{serviceName}.{ns}.svc.cluster.local:{port}"; logger.LogInformation("Confirmed Prometheus at {Url}", prometheusUrl); confirmedEndpoints.Add(new PrometheusEndpoint(prometheusUrl, ns, serviceName)); // Don't check other ports on the same service — we found it. break; } } } logger.LogInformation("Found {Count} Prometheus instance(s) in cluster {ClusterName}", confirmedEndpoints.Count, cluster.Name); return confirmedEndpoints; } /// /// Determines which ports on a service are candidates for being Prometheus. /// We prioritize well-known ports (9090) and HTTP-named ports, but also /// include any port in the 8000-9999 range as a fallback since Prometheus /// can be configured on non-standard ports. /// private static List GetCandidatePorts(V1Service service) { List ports = new(); if (service.Spec?.Ports is null) { return ports; } foreach (V1ServicePort servicePort in service.Spec.Ports) { int port = servicePort.Port; string? portName = servicePort.Name?.ToLowerInvariant(); // Priority 1: Well-known Prometheus port if (port == 9090) { ports.Insert(0, port); continue; } // Priority 2: Ports with HTTP-like names (common in Prometheus deployments) if (portName is "http" or "http-web" or "web" or "http-metrics" or "metrics") { ports.Add(port); continue; } // Priority 3: Ports in the typical application range if (port is >= 8080 and <= 9999) { ports.Add(port); } } return ports; } /// /// Attempts to reach a service through the Kubernetes API service proxy and verify /// it's actually Prometheus. The K8s API proxy lets us access cluster-internal /// services without needing port-forwarding or direct network access. /// /// Proxy URL format: /// {apiserver}/api/v1/namespaces/{ns}/services/{svc}:{port}/proxy/{path} /// /// Verification strategy: /// 1. Call /-/ready — Prometheus returns 200 with "Prometheus Server is Ready." /// 2. Call /api/v1/query?query=vector(1) — only Prometheus can evaluate PromQL /// 3. Call /-/healthy — fallback for older versions /// private async Task ProbeServiceForPrometheusAsync( Kubernetes client, Uri baseUri, string ns, string serviceName, int port, CancellationToken ct) { try { // Create a per-probe cancellation token with timeout so one slow service // doesn't block the entire scan. using CancellationTokenSource timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); timeoutCts.CancelAfter(ProbeTimeout); CancellationToken probeCt = timeoutCts.Token; HttpClient httpClient = client.HttpClient; // Construct absolute URIs to avoid relative URI resolution issues. // The Kubernetes service proxy path is well-defined. string proxyBase = $"{baseUri.ToString().TrimEnd('/')}/api/v1/namespaces/{ns}/services/{serviceName}:{port}/proxy"; // Strategy 1: Try /-/ready endpoint. // Prometheus 2.x+ responds with 200 and body "Prometheus Server is Ready.\n" Uri readyUri = new($"{proxyBase}/-/ready"); using (HttpRequestMessage readyRequest = new(HttpMethod.Get, readyUri)) { using HttpResponseMessage readyResponse = await httpClient.SendAsync(readyRequest, probeCt); logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/ready => {StatusCode}", serviceName, port, ns, (int)readyResponse.StatusCode); if (readyResponse.IsSuccessStatusCode) { string readyBody = await readyResponse.Content.ReadAsStringAsync(probeCt); if (readyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase)) { logger.LogInformation("Confirmed via /-/ready: {Service}:{Port} in {Namespace}", serviceName, port, ns); return true; } logger.LogInformation("/-/ready returned 200 but body was: {Body}", readyBody.Trim()); } } // Strategy 2: Try a PromQL query that only Prometheus can answer. // vector(1) is a trivial PromQL expression that returns a single instant vector. Uri queryUri = new($"{proxyBase}/api/v1/query?query=vector(1)"); using (HttpRequestMessage queryRequest = new(HttpMethod.Get, queryUri)) { using HttpResponseMessage queryResponse = await httpClient.SendAsync(queryRequest, probeCt); logger.LogInformation("Probe {Service}:{Port} in {Namespace} /api/v1/query => {StatusCode}", serviceName, port, ns, (int)queryResponse.StatusCode); if (queryResponse.IsSuccessStatusCode) { string queryBody = await queryResponse.Content.ReadAsStringAsync(probeCt); // A valid Prometheus response contains "status":"success" and "resultType". if (queryBody.Contains("\"status\":\"success\"", StringComparison.Ordinal) && queryBody.Contains("\"resultType\"", StringComparison.Ordinal)) { logger.LogInformation("Confirmed via PromQL query: {Service}:{Port} in {Namespace}", serviceName, port, ns); return true; } logger.LogInformation("/api/v1/query returned 200 but body was: {Body}", queryBody[..Math.Min(200, queryBody.Length)]); } } // Strategy 3: Try /-/healthy as a last resort. Uri healthyUri = new($"{proxyBase}/-/healthy"); using (HttpRequestMessage healthyRequest = new(HttpMethod.Get, healthyUri)) { using HttpResponseMessage healthyResponse = await httpClient.SendAsync(healthyRequest, probeCt); logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/healthy => {StatusCode}", serviceName, port, ns, (int)healthyResponse.StatusCode); if (healthyResponse.IsSuccessStatusCode) { string healthyBody = await healthyResponse.Content.ReadAsStringAsync(probeCt); if (healthyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase)) { logger.LogInformation("Confirmed via /-/healthy: {Service}:{Port} in {Namespace}", serviceName, port, ns); return true; } logger.LogInformation("/-/healthy returned 200 but body was: {Body}", healthyBody.Trim()); } } } catch (OperationCanceledException) when (!ct.IsCancellationRequested) { // Per-probe timeout — service didn't respond fast enough, skip it. logger.LogInformation("Timeout probing {Service}:{Port} in {Namespace} (>{TimeoutSeconds}s)", serviceName, port, ns, ProbeTimeout.TotalSeconds); } catch (HttpRequestException ex) { // Service is not reachable via proxy. logger.LogInformation("HTTP error probing {Service}:{Port} in {Namespace}: {Message}", serviceName, port, ns, ex.Message); } catch (Exception ex) { logger.LogWarning(ex, "Unexpected error probing {Service}:{Port} in {Namespace}", serviceName, port, ns); } return false; } /// /// Creates a Kubernetes client from the cluster's stored kubeconfig, /// selecting the specific context that was chosen during registration. /// private static Kubernetes BuildClient(KubernetesCluster cluster) { // Write kubeconfig to a temporary stream for parsing. using MemoryStream stream = new(Encoding.UTF8.GetBytes(cluster.KubeConfig)); KubernetesClientConfiguration config = KubernetesClientConfiguration.BuildConfigFromConfigFile( stream, currentContext: cluster.ContextName); return new Kubernetes(config); } }