342 lines
13 KiB
C#
342 lines
13 KiB
C#
using System.Text;
|
|
using EntKube.Clusters.Domain;
|
|
using k8s;
|
|
using k8s.Models;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace EntKube.Clusters.Features.DiscoverPrometheus;
|
|
|
|
/// <summary>
|
|
/// Scans a Kubernetes cluster for actual running Prometheus instances.
|
|
/// We cannot rely on service names or labels alone — anyone can name a service
|
|
/// "prometheus" without it being one. Instead, this scanner:
|
|
///
|
|
/// 1. Connects to the cluster using the stored kubeconfig
|
|
/// 2. Lists ALL services across all namespaces
|
|
/// 3. For each service with an HTTP port, proxies through the Kubernetes API
|
|
/// to call the service and verify it responds as Prometheus
|
|
/// 4. Confirms the service is Prometheus by calling /-/ready or /api/v1/query
|
|
///
|
|
/// This approach finds Prometheus regardless of how it was deployed (Helm,
|
|
/// operator, manual manifests, custom names) because we verify the actual
|
|
/// running software, not metadata about it.
|
|
/// </summary>
|
|
public class KubernetesPrometheusScanner : IPrometheusScanner
|
|
{
|
|
private readonly ILogger<KubernetesPrometheusScanner> logger;
|
|
|
|
// Per-service probe timeout — don't wait too long on unresponsive services.
|
|
|
|
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5);
|
|
|
|
public KubernetesPrometheusScanner(ILogger<KubernetesPrometheusScanner> logger)
|
|
{
|
|
this.logger = logger;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Scans all namespaces by listing every service in the cluster, then
|
|
/// probing each one through the Kubernetes API proxy to confirm whether
|
|
/// it's actually a Prometheus instance. Returns all confirmed endpoints.
|
|
/// </summary>
|
|
public async Task<List<PrometheusEndpoint>> ScanAsync(KubernetesCluster cluster, CancellationToken ct = default)
|
|
{
|
|
// Build a Kubernetes client from the cluster's stored kubeconfig.
|
|
// We use the specific context that was selected during registration.
|
|
|
|
Kubernetes client = BuildClient(cluster);
|
|
List<PrometheusEndpoint> confirmedEndpoints = new();
|
|
|
|
// Get the base URI so we can construct absolute proxy URLs.
|
|
// Using absolute URIs avoids subtle bugs with relative URI resolution
|
|
// where BaseAddress trailing-slash presence changes the behavior.
|
|
|
|
Uri baseUri = client.BaseUri;
|
|
logger.LogInformation("Connecting to cluster {ClusterName} at {BaseUri}", cluster.Name, baseUri);
|
|
|
|
// List all services across all namespaces. We don't filter by labels
|
|
// because we want to find Prometheus instances regardless of how they
|
|
// were deployed or labelled.
|
|
|
|
V1ServiceList serviceList;
|
|
|
|
try
|
|
{
|
|
serviceList = await client.CoreV1.ListServiceForAllNamespacesAsync(cancellationToken: ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogWarning(ex, "Failed to list services in cluster {ClusterName}. Is the cluster reachable?", cluster.Name);
|
|
return confirmedEndpoints;
|
|
}
|
|
|
|
logger.LogInformation("Scanning {ServiceCount} services in cluster {ClusterName} for Prometheus instances",
|
|
serviceList.Items.Count, cluster.Name);
|
|
|
|
// For each service, check if any of its ports could be Prometheus.
|
|
// We focus on HTTP-like ports (named http, web, or common Prometheus ports).
|
|
|
|
foreach (V1Service service in serviceList.Items)
|
|
{
|
|
if (ct.IsCancellationRequested)
|
|
{
|
|
break;
|
|
}
|
|
|
|
string ns = service.Metadata.NamespaceProperty ?? "default";
|
|
string serviceName = service.Metadata.Name;
|
|
|
|
// Skip the kubernetes API service itself.
|
|
|
|
if (serviceName == "kubernetes" && ns == "default")
|
|
{
|
|
continue;
|
|
}
|
|
|
|
List<int> candidatePorts = GetCandidatePorts(service);
|
|
|
|
if (candidatePorts.Count == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
logger.LogInformation("Probing {Service} in {Namespace} on ports [{Ports}]",
|
|
serviceName, ns, string.Join(", ", candidatePorts));
|
|
|
|
foreach (int port in candidatePorts)
|
|
{
|
|
if (ct.IsCancellationRequested)
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Try to proxy through the Kubernetes API to reach this service
|
|
// and verify it's Prometheus by calling its endpoints.
|
|
|
|
bool isPrometheus = await ProbeServiceForPrometheusAsync(client, baseUri, ns, serviceName, port, ct);
|
|
|
|
if (isPrometheus)
|
|
{
|
|
// Build the in-cluster URL that other services can use to reach this Prometheus.
|
|
|
|
string prometheusUrl = $"http://{serviceName}.{ns}.svc.cluster.local:{port}";
|
|
|
|
logger.LogInformation("Confirmed Prometheus at {Url}", prometheusUrl);
|
|
|
|
confirmedEndpoints.Add(new PrometheusEndpoint(prometheusUrl, ns, serviceName));
|
|
|
|
// Don't check other ports on the same service — we found it.
|
|
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.LogInformation("Found {Count} Prometheus instance(s) in cluster {ClusterName}",
|
|
confirmedEndpoints.Count, cluster.Name);
|
|
|
|
return confirmedEndpoints;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Determines which ports on a service are candidates for being Prometheus.
|
|
/// We prioritize well-known ports (9090) and HTTP-named ports, but also
|
|
/// include any port in the 8000-9999 range as a fallback since Prometheus
|
|
/// can be configured on non-standard ports.
|
|
/// </summary>
|
|
private static List<int> GetCandidatePorts(V1Service service)
|
|
{
|
|
List<int> ports = new();
|
|
|
|
if (service.Spec?.Ports is null)
|
|
{
|
|
return ports;
|
|
}
|
|
|
|
foreach (V1ServicePort servicePort in service.Spec.Ports)
|
|
{
|
|
int port = servicePort.Port;
|
|
string? portName = servicePort.Name?.ToLowerInvariant();
|
|
|
|
// Priority 1: Well-known Prometheus port
|
|
|
|
if (port == 9090)
|
|
{
|
|
ports.Insert(0, port);
|
|
continue;
|
|
}
|
|
|
|
// Priority 2: Ports with HTTP-like names (common in Prometheus deployments)
|
|
|
|
if (portName is "http" or "http-web" or "web" or "http-metrics" or "metrics")
|
|
{
|
|
ports.Add(port);
|
|
continue;
|
|
}
|
|
|
|
// Priority 3: Ports in the typical application range
|
|
|
|
if (port is >= 8080 and <= 9999)
|
|
{
|
|
ports.Add(port);
|
|
}
|
|
}
|
|
|
|
return ports;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Attempts to reach a service through the Kubernetes API service proxy and verify
|
|
/// it's actually Prometheus. The K8s API proxy lets us access cluster-internal
|
|
/// services without needing port-forwarding or direct network access.
|
|
///
|
|
/// Proxy URL format:
|
|
/// {apiserver}/api/v1/namespaces/{ns}/services/{svc}:{port}/proxy/{path}
|
|
///
|
|
/// Verification strategy:
|
|
/// 1. Call /-/ready — Prometheus returns 200 with "Prometheus Server is Ready."
|
|
/// 2. Call /api/v1/query?query=vector(1) — only Prometheus can evaluate PromQL
|
|
/// 3. Call /-/healthy — fallback for older versions
|
|
/// </summary>
|
|
private async Task<bool> ProbeServiceForPrometheusAsync(
|
|
Kubernetes client,
|
|
Uri baseUri,
|
|
string ns,
|
|
string serviceName,
|
|
int port,
|
|
CancellationToken ct)
|
|
{
|
|
try
|
|
{
|
|
// Create a per-probe cancellation token with timeout so one slow service
|
|
// doesn't block the entire scan.
|
|
|
|
using CancellationTokenSource timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
|
timeoutCts.CancelAfter(ProbeTimeout);
|
|
CancellationToken probeCt = timeoutCts.Token;
|
|
|
|
HttpClient httpClient = client.HttpClient;
|
|
|
|
// Construct absolute URIs to avoid relative URI resolution issues.
|
|
// The Kubernetes service proxy path is well-defined.
|
|
|
|
string proxyBase = $"{baseUri.ToString().TrimEnd('/')}/api/v1/namespaces/{ns}/services/{serviceName}:{port}/proxy";
|
|
|
|
// Strategy 1: Try /-/ready endpoint.
|
|
// Prometheus 2.x+ responds with 200 and body "Prometheus Server is Ready.\n"
|
|
|
|
Uri readyUri = new($"{proxyBase}/-/ready");
|
|
|
|
using (HttpRequestMessage readyRequest = new(HttpMethod.Get, readyUri))
|
|
{
|
|
using HttpResponseMessage readyResponse = await httpClient.SendAsync(readyRequest, probeCt);
|
|
|
|
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/ready => {StatusCode}",
|
|
serviceName, port, ns, (int)readyResponse.StatusCode);
|
|
|
|
if (readyResponse.IsSuccessStatusCode)
|
|
{
|
|
string readyBody = await readyResponse.Content.ReadAsStringAsync(probeCt);
|
|
|
|
if (readyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
logger.LogInformation("Confirmed via /-/ready: {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
|
return true;
|
|
}
|
|
|
|
logger.LogInformation("/-/ready returned 200 but body was: {Body}", readyBody.Trim());
|
|
}
|
|
}
|
|
|
|
// Strategy 2: Try a PromQL query that only Prometheus can answer.
|
|
// vector(1) is a trivial PromQL expression that returns a single instant vector.
|
|
|
|
Uri queryUri = new($"{proxyBase}/api/v1/query?query=vector(1)");
|
|
|
|
using (HttpRequestMessage queryRequest = new(HttpMethod.Get, queryUri))
|
|
{
|
|
using HttpResponseMessage queryResponse = await httpClient.SendAsync(queryRequest, probeCt);
|
|
|
|
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /api/v1/query => {StatusCode}",
|
|
serviceName, port, ns, (int)queryResponse.StatusCode);
|
|
|
|
if (queryResponse.IsSuccessStatusCode)
|
|
{
|
|
string queryBody = await queryResponse.Content.ReadAsStringAsync(probeCt);
|
|
|
|
// A valid Prometheus response contains "status":"success" and "resultType".
|
|
|
|
if (queryBody.Contains("\"status\":\"success\"", StringComparison.Ordinal) &&
|
|
queryBody.Contains("\"resultType\"", StringComparison.Ordinal))
|
|
{
|
|
logger.LogInformation("Confirmed via PromQL query: {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
|
return true;
|
|
}
|
|
|
|
logger.LogInformation("/api/v1/query returned 200 but body was: {Body}", queryBody[..Math.Min(200, queryBody.Length)]);
|
|
}
|
|
}
|
|
|
|
// Strategy 3: Try /-/healthy as a last resort.
|
|
|
|
Uri healthyUri = new($"{proxyBase}/-/healthy");
|
|
|
|
using (HttpRequestMessage healthyRequest = new(HttpMethod.Get, healthyUri))
|
|
{
|
|
using HttpResponseMessage healthyResponse = await httpClient.SendAsync(healthyRequest, probeCt);
|
|
|
|
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/healthy => {StatusCode}",
|
|
serviceName, port, ns, (int)healthyResponse.StatusCode);
|
|
|
|
if (healthyResponse.IsSuccessStatusCode)
|
|
{
|
|
string healthyBody = await healthyResponse.Content.ReadAsStringAsync(probeCt);
|
|
|
|
if (healthyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
logger.LogInformation("Confirmed via /-/healthy: {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
|
return true;
|
|
}
|
|
|
|
logger.LogInformation("/-/healthy returned 200 but body was: {Body}", healthyBody.Trim());
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
|
{
|
|
// Per-probe timeout — service didn't respond fast enough, skip it.
|
|
logger.LogInformation("Timeout probing {Service}:{Port} in {Namespace} (>{TimeoutSeconds}s)",
|
|
serviceName, port, ns, ProbeTimeout.TotalSeconds);
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
// Service is not reachable via proxy.
|
|
logger.LogInformation("HTTP error probing {Service}:{Port} in {Namespace}: {Message}", serviceName, port, ns, ex.Message);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogWarning(ex, "Unexpected error probing {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a Kubernetes client from the cluster's stored kubeconfig,
|
|
/// selecting the specific context that was chosen during registration.
|
|
/// </summary>
|
|
private static Kubernetes BuildClient(KubernetesCluster cluster)
|
|
{
|
|
// Write kubeconfig to a temporary stream for parsing.
|
|
|
|
using MemoryStream stream = new(Encoding.UTF8.GetBytes(cluster.KubeConfig));
|
|
KubernetesClientConfiguration config = KubernetesClientConfiguration.BuildConfigFromConfigFile(
|
|
stream,
|
|
currentContext: cluster.ContextName);
|
|
|
|
return new Kubernetes(config);
|
|
}
|
|
}
|
|
|
|
|