using System.Text;
using EntKube.Clusters.Domain;
using k8s;
using k8s.Models;
using Microsoft.Extensions.Logging;
namespace EntKube.Clusters.Features.DiscoverPrometheus;
///
/// Scans a Kubernetes cluster for actual running Prometheus instances.
/// We cannot rely on service names or labels alone — anyone can name a service
/// "prometheus" without it being one. Instead, this scanner:
///
/// 1. Connects to the cluster using the stored kubeconfig
/// 2. Lists ALL services across all namespaces
/// 3. For each service with an HTTP port, proxies through the Kubernetes API
/// to call the service and verify it responds as Prometheus
/// 4. Confirms the service is Prometheus by calling /-/ready or /api/v1/query
///
/// This approach finds Prometheus regardless of how it was deployed (Helm,
/// operator, manual manifests, custom names) because we verify the actual
/// running software, not metadata about it.
///
public class KubernetesPrometheusScanner : IPrometheusScanner
{
private readonly ILogger logger;
// Per-service probe timeout — don't wait too long on unresponsive services.
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5);
public KubernetesPrometheusScanner(ILogger logger)
{
this.logger = logger;
}
///
/// Scans all namespaces by listing every service in the cluster, then
/// probing each one through the Kubernetes API proxy to confirm whether
/// it's actually a Prometheus instance. Returns all confirmed endpoints.
///
public async Task> ScanAsync(KubernetesCluster cluster, CancellationToken ct = default)
{
// Build a Kubernetes client from the cluster's stored kubeconfig.
// We use the specific context that was selected during registration.
Kubernetes client = BuildClient(cluster);
List confirmedEndpoints = new();
// Get the base URI so we can construct absolute proxy URLs.
// Using absolute URIs avoids subtle bugs with relative URI resolution
// where BaseAddress trailing-slash presence changes the behavior.
Uri baseUri = client.BaseUri;
logger.LogInformation("Connecting to cluster {ClusterName} at {BaseUri}", cluster.Name, baseUri);
// List all services across all namespaces. We don't filter by labels
// because we want to find Prometheus instances regardless of how they
// were deployed or labelled.
V1ServiceList serviceList;
try
{
serviceList = await client.CoreV1.ListServiceForAllNamespacesAsync(cancellationToken: ct);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to list services in cluster {ClusterName}. Is the cluster reachable?", cluster.Name);
return confirmedEndpoints;
}
logger.LogInformation("Scanning {ServiceCount} services in cluster {ClusterName} for Prometheus instances",
serviceList.Items.Count, cluster.Name);
// For each service, check if any of its ports could be Prometheus.
// We focus on HTTP-like ports (named http, web, or common Prometheus ports).
foreach (V1Service service in serviceList.Items)
{
if (ct.IsCancellationRequested)
{
break;
}
string ns = service.Metadata.NamespaceProperty ?? "default";
string serviceName = service.Metadata.Name;
// Skip the kubernetes API service itself.
if (serviceName == "kubernetes" && ns == "default")
{
continue;
}
List candidatePorts = GetCandidatePorts(service);
if (candidatePorts.Count == 0)
{
continue;
}
logger.LogInformation("Probing {Service} in {Namespace} on ports [{Ports}]",
serviceName, ns, string.Join(", ", candidatePorts));
foreach (int port in candidatePorts)
{
if (ct.IsCancellationRequested)
{
break;
}
// Try to proxy through the Kubernetes API to reach this service
// and verify it's Prometheus by calling its endpoints.
bool isPrometheus = await ProbeServiceForPrometheusAsync(client, baseUri, ns, serviceName, port, ct);
if (isPrometheus)
{
// Build the in-cluster URL that other services can use to reach this Prometheus.
string prometheusUrl = $"http://{serviceName}.{ns}.svc.cluster.local:{port}";
logger.LogInformation("Confirmed Prometheus at {Url}", prometheusUrl);
confirmedEndpoints.Add(new PrometheusEndpoint(prometheusUrl, ns, serviceName));
// Don't check other ports on the same service — we found it.
break;
}
}
}
logger.LogInformation("Found {Count} Prometheus instance(s) in cluster {ClusterName}",
confirmedEndpoints.Count, cluster.Name);
return confirmedEndpoints;
}
///
/// Determines which ports on a service are candidates for being Prometheus.
/// We prioritize well-known ports (9090) and HTTP-named ports, but also
/// include any port in the 8000-9999 range as a fallback since Prometheus
/// can be configured on non-standard ports.
///
private static List GetCandidatePorts(V1Service service)
{
List ports = new();
if (service.Spec?.Ports is null)
{
return ports;
}
foreach (V1ServicePort servicePort in service.Spec.Ports)
{
int port = servicePort.Port;
string? portName = servicePort.Name?.ToLowerInvariant();
// Priority 1: Well-known Prometheus port
if (port == 9090)
{
ports.Insert(0, port);
continue;
}
// Priority 2: Ports with HTTP-like names (common in Prometheus deployments)
if (portName is "http" or "http-web" or "web" or "http-metrics" or "metrics")
{
ports.Add(port);
continue;
}
// Priority 3: Ports in the typical application range
if (port is >= 8080 and <= 9999)
{
ports.Add(port);
}
}
return ports;
}
///
/// Attempts to reach a service through the Kubernetes API service proxy and verify
/// it's actually Prometheus. The K8s API proxy lets us access cluster-internal
/// services without needing port-forwarding or direct network access.
///
/// Proxy URL format:
/// {apiserver}/api/v1/namespaces/{ns}/services/{svc}:{port}/proxy/{path}
///
/// Verification strategy:
/// 1. Call /-/ready — Prometheus returns 200 with "Prometheus Server is Ready."
/// 2. Call /api/v1/query?query=vector(1) — only Prometheus can evaluate PromQL
/// 3. Call /-/healthy — fallback for older versions
///
private async Task ProbeServiceForPrometheusAsync(
Kubernetes client,
Uri baseUri,
string ns,
string serviceName,
int port,
CancellationToken ct)
{
try
{
// Create a per-probe cancellation token with timeout so one slow service
// doesn't block the entire scan.
using CancellationTokenSource timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
timeoutCts.CancelAfter(ProbeTimeout);
CancellationToken probeCt = timeoutCts.Token;
HttpClient httpClient = client.HttpClient;
// Construct absolute URIs to avoid relative URI resolution issues.
// The Kubernetes service proxy path is well-defined.
string proxyBase = $"{baseUri.ToString().TrimEnd('/')}/api/v1/namespaces/{ns}/services/{serviceName}:{port}/proxy";
// Strategy 1: Try /-/ready endpoint.
// Prometheus 2.x+ responds with 200 and body "Prometheus Server is Ready.\n"
Uri readyUri = new($"{proxyBase}/-/ready");
using (HttpRequestMessage readyRequest = new(HttpMethod.Get, readyUri))
{
using HttpResponseMessage readyResponse = await httpClient.SendAsync(readyRequest, probeCt);
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/ready => {StatusCode}",
serviceName, port, ns, (int)readyResponse.StatusCode);
if (readyResponse.IsSuccessStatusCode)
{
string readyBody = await readyResponse.Content.ReadAsStringAsync(probeCt);
if (readyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase))
{
logger.LogInformation("Confirmed via /-/ready: {Service}:{Port} in {Namespace}", serviceName, port, ns);
return true;
}
logger.LogInformation("/-/ready returned 200 but body was: {Body}", readyBody.Trim());
}
}
// Strategy 2: Try a PromQL query that only Prometheus can answer.
// vector(1) is a trivial PromQL expression that returns a single instant vector.
Uri queryUri = new($"{proxyBase}/api/v1/query?query=vector(1)");
using (HttpRequestMessage queryRequest = new(HttpMethod.Get, queryUri))
{
using HttpResponseMessage queryResponse = await httpClient.SendAsync(queryRequest, probeCt);
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /api/v1/query => {StatusCode}",
serviceName, port, ns, (int)queryResponse.StatusCode);
if (queryResponse.IsSuccessStatusCode)
{
string queryBody = await queryResponse.Content.ReadAsStringAsync(probeCt);
// A valid Prometheus response contains "status":"success" and "resultType".
if (queryBody.Contains("\"status\":\"success\"", StringComparison.Ordinal) &&
queryBody.Contains("\"resultType\"", StringComparison.Ordinal))
{
logger.LogInformation("Confirmed via PromQL query: {Service}:{Port} in {Namespace}", serviceName, port, ns);
return true;
}
logger.LogInformation("/api/v1/query returned 200 but body was: {Body}", queryBody[..Math.Min(200, queryBody.Length)]);
}
}
// Strategy 3: Try /-/healthy as a last resort.
Uri healthyUri = new($"{proxyBase}/-/healthy");
using (HttpRequestMessage healthyRequest = new(HttpMethod.Get, healthyUri))
{
using HttpResponseMessage healthyResponse = await httpClient.SendAsync(healthyRequest, probeCt);
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/healthy => {StatusCode}",
serviceName, port, ns, (int)healthyResponse.StatusCode);
if (healthyResponse.IsSuccessStatusCode)
{
string healthyBody = await healthyResponse.Content.ReadAsStringAsync(probeCt);
if (healthyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase))
{
logger.LogInformation("Confirmed via /-/healthy: {Service}:{Port} in {Namespace}", serviceName, port, ns);
return true;
}
logger.LogInformation("/-/healthy returned 200 but body was: {Body}", healthyBody.Trim());
}
}
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
// Per-probe timeout — service didn't respond fast enough, skip it.
logger.LogInformation("Timeout probing {Service}:{Port} in {Namespace} (>{TimeoutSeconds}s)",
serviceName, port, ns, ProbeTimeout.TotalSeconds);
}
catch (HttpRequestException ex)
{
// Service is not reachable via proxy.
logger.LogInformation("HTTP error probing {Service}:{Port} in {Namespace}: {Message}", serviceName, port, ns, ex.Message);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Unexpected error probing {Service}:{Port} in {Namespace}", serviceName, port, ns);
}
return false;
}
///
/// Creates a Kubernetes client from the cluster's stored kubeconfig,
/// selecting the specific context that was chosen during registration.
///
private static Kubernetes BuildClient(KubernetesCluster cluster)
{
// Write kubeconfig to a temporary stream for parsing.
using MemoryStream stream = new(Encoding.UTF8.GetBytes(cluster.KubeConfig));
KubernetesClientConfiguration config = KubernetesClientConfiguration.BuildConfigFromConfigFile(
stream,
currentContext: cluster.ContextName);
return new Kubernetes(config);
}
}