too much for one commit
This commit is contained in:
Nils Blomgren
@@ -0,0 +1,341 @@
|
||||
using System.Text;
|
||||
using EntKube.Clusters.Domain;
|
||||
using k8s;
|
||||
using k8s.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace EntKube.Clusters.Features.DiscoverPrometheus;
|
||||
|
||||
/// <summary>
|
||||
/// Scans a Kubernetes cluster for actual running Prometheus instances.
|
||||
/// We cannot rely on service names or labels alone — anyone can name a service
|
||||
/// "prometheus" without it being one. Instead, this scanner:
|
||||
///
|
||||
/// 1. Connects to the cluster using the stored kubeconfig
|
||||
/// 2. Lists ALL services across all namespaces
|
||||
/// 3. For each service with an HTTP port, proxies through the Kubernetes API
|
||||
/// to call the service and verify it responds as Prometheus
|
||||
/// 4. Confirms the service is Prometheus by calling /-/ready or /api/v1/query
|
||||
///
|
||||
/// This approach finds Prometheus regardless of how it was deployed (Helm,
|
||||
/// operator, manual manifests, custom names) because we verify the actual
|
||||
/// running software, not metadata about it.
|
||||
/// </summary>
|
||||
public class KubernetesPrometheusScanner : IPrometheusScanner
|
||||
{
|
||||
private readonly ILogger<KubernetesPrometheusScanner> logger;
|
||||
|
||||
// Per-service probe timeout — don't wait too long on unresponsive services.
|
||||
|
||||
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
public KubernetesPrometheusScanner(ILogger<KubernetesPrometheusScanner> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scans all namespaces by listing every service in the cluster, then
|
||||
/// probing each one through the Kubernetes API proxy to confirm whether
|
||||
/// it's actually a Prometheus instance. Returns all confirmed endpoints.
|
||||
/// </summary>
|
||||
public async Task<List<PrometheusEndpoint>> ScanAsync(KubernetesCluster cluster, CancellationToken ct = default)
|
||||
{
|
||||
// Build a Kubernetes client from the cluster's stored kubeconfig.
|
||||
// We use the specific context that was selected during registration.
|
||||
|
||||
Kubernetes client = BuildClient(cluster);
|
||||
List<PrometheusEndpoint> confirmedEndpoints = new();
|
||||
|
||||
// Get the base URI so we can construct absolute proxy URLs.
|
||||
// Using absolute URIs avoids subtle bugs with relative URI resolution
|
||||
// where BaseAddress trailing-slash presence changes the behavior.
|
||||
|
||||
Uri baseUri = client.BaseUri;
|
||||
logger.LogInformation("Connecting to cluster {ClusterName} at {BaseUri}", cluster.Name, baseUri);
|
||||
|
||||
// List all services across all namespaces. We don't filter by labels
|
||||
// because we want to find Prometheus instances regardless of how they
|
||||
// were deployed or labelled.
|
||||
|
||||
V1ServiceList serviceList;
|
||||
|
||||
try
|
||||
{
|
||||
serviceList = await client.CoreV1.ListServiceForAllNamespacesAsync(cancellationToken: ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Failed to list services in cluster {ClusterName}. Is the cluster reachable?", cluster.Name);
|
||||
return confirmedEndpoints;
|
||||
}
|
||||
|
||||
logger.LogInformation("Scanning {ServiceCount} services in cluster {ClusterName} for Prometheus instances",
|
||||
serviceList.Items.Count, cluster.Name);
|
||||
|
||||
// For each service, check if any of its ports could be Prometheus.
|
||||
// We focus on HTTP-like ports (named http, web, or common Prometheus ports).
|
||||
|
||||
foreach (V1Service service in serviceList.Items)
|
||||
{
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
string ns = service.Metadata.NamespaceProperty ?? "default";
|
||||
string serviceName = service.Metadata.Name;
|
||||
|
||||
// Skip the kubernetes API service itself.
|
||||
|
||||
if (serviceName == "kubernetes" && ns == "default")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
List<int> candidatePorts = GetCandidatePorts(service);
|
||||
|
||||
if (candidatePorts.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.LogInformation("Probing {Service} in {Namespace} on ports [{Ports}]",
|
||||
serviceName, ns, string.Join(", ", candidatePorts));
|
||||
|
||||
foreach (int port in candidatePorts)
|
||||
{
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Try to proxy through the Kubernetes API to reach this service
|
||||
// and verify it's Prometheus by calling its endpoints.
|
||||
|
||||
bool isPrometheus = await ProbeServiceForPrometheusAsync(client, baseUri, ns, serviceName, port, ct);
|
||||
|
||||
if (isPrometheus)
|
||||
{
|
||||
// Build the in-cluster URL that other services can use to reach this Prometheus.
|
||||
|
||||
string prometheusUrl = $"http://{serviceName}.{ns}.svc.cluster.local:{port}";
|
||||
|
||||
logger.LogInformation("Confirmed Prometheus at {Url}", prometheusUrl);
|
||||
|
||||
confirmedEndpoints.Add(new PrometheusEndpoint(prometheusUrl, ns, serviceName));
|
||||
|
||||
// Don't check other ports on the same service — we found it.
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation("Found {Count} Prometheus instance(s) in cluster {ClusterName}",
|
||||
confirmedEndpoints.Count, cluster.Name);
|
||||
|
||||
return confirmedEndpoints;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines which ports on a service are candidates for being Prometheus.
|
||||
/// We prioritize well-known ports (9090) and HTTP-named ports, but also
|
||||
/// include any port in the 8000-9999 range as a fallback since Prometheus
|
||||
/// can be configured on non-standard ports.
|
||||
/// </summary>
|
||||
private static List<int> GetCandidatePorts(V1Service service)
|
||||
{
|
||||
List<int> ports = new();
|
||||
|
||||
if (service.Spec?.Ports is null)
|
||||
{
|
||||
return ports;
|
||||
}
|
||||
|
||||
foreach (V1ServicePort servicePort in service.Spec.Ports)
|
||||
{
|
||||
int port = servicePort.Port;
|
||||
string? portName = servicePort.Name?.ToLowerInvariant();
|
||||
|
||||
// Priority 1: Well-known Prometheus port
|
||||
|
||||
if (port == 9090)
|
||||
{
|
||||
ports.Insert(0, port);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Priority 2: Ports with HTTP-like names (common in Prometheus deployments)
|
||||
|
||||
if (portName is "http" or "http-web" or "web" or "http-metrics" or "metrics")
|
||||
{
|
||||
ports.Add(port);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Priority 3: Ports in the typical application range
|
||||
|
||||
if (port is >= 8080 and <= 9999)
|
||||
{
|
||||
ports.Add(port);
|
||||
}
|
||||
}
|
||||
|
||||
return ports;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to reach a service through the Kubernetes API service proxy and verify
|
||||
/// it's actually Prometheus. The K8s API proxy lets us access cluster-internal
|
||||
/// services without needing port-forwarding or direct network access.
|
||||
///
|
||||
/// Proxy URL format:
|
||||
/// {apiserver}/api/v1/namespaces/{ns}/services/{svc}:{port}/proxy/{path}
|
||||
///
|
||||
/// Verification strategy:
|
||||
/// 1. Call /-/ready — Prometheus returns 200 with "Prometheus Server is Ready."
|
||||
/// 2. Call /api/v1/query?query=vector(1) — only Prometheus can evaluate PromQL
|
||||
/// 3. Call /-/healthy — fallback for older versions
|
||||
/// </summary>
|
||||
private async Task<bool> ProbeServiceForPrometheusAsync(
|
||||
Kubernetes client,
|
||||
Uri baseUri,
|
||||
string ns,
|
||||
string serviceName,
|
||||
int port,
|
||||
CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Create a per-probe cancellation token with timeout so one slow service
|
||||
// doesn't block the entire scan.
|
||||
|
||||
using CancellationTokenSource timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
timeoutCts.CancelAfter(ProbeTimeout);
|
||||
CancellationToken probeCt = timeoutCts.Token;
|
||||
|
||||
HttpClient httpClient = client.HttpClient;
|
||||
|
||||
// Construct absolute URIs to avoid relative URI resolution issues.
|
||||
// The Kubernetes service proxy path is well-defined.
|
||||
|
||||
string proxyBase = $"{baseUri.ToString().TrimEnd('/')}/api/v1/namespaces/{ns}/services/{serviceName}:{port}/proxy";
|
||||
|
||||
// Strategy 1: Try /-/ready endpoint.
|
||||
// Prometheus 2.x+ responds with 200 and body "Prometheus Server is Ready.\n"
|
||||
|
||||
Uri readyUri = new($"{proxyBase}/-/ready");
|
||||
|
||||
using (HttpRequestMessage readyRequest = new(HttpMethod.Get, readyUri))
|
||||
{
|
||||
using HttpResponseMessage readyResponse = await httpClient.SendAsync(readyRequest, probeCt);
|
||||
|
||||
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/ready => {StatusCode}",
|
||||
serviceName, port, ns, (int)readyResponse.StatusCode);
|
||||
|
||||
if (readyResponse.IsSuccessStatusCode)
|
||||
{
|
||||
string readyBody = await readyResponse.Content.ReadAsStringAsync(probeCt);
|
||||
|
||||
if (readyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
logger.LogInformation("Confirmed via /-/ready: {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
||||
return true;
|
||||
}
|
||||
|
||||
logger.LogInformation("/-/ready returned 200 but body was: {Body}", readyBody.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 2: Try a PromQL query that only Prometheus can answer.
|
||||
// vector(1) is a trivial PromQL expression that returns a single instant vector.
|
||||
|
||||
Uri queryUri = new($"{proxyBase}/api/v1/query?query=vector(1)");
|
||||
|
||||
using (HttpRequestMessage queryRequest = new(HttpMethod.Get, queryUri))
|
||||
{
|
||||
using HttpResponseMessage queryResponse = await httpClient.SendAsync(queryRequest, probeCt);
|
||||
|
||||
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /api/v1/query => {StatusCode}",
|
||||
serviceName, port, ns, (int)queryResponse.StatusCode);
|
||||
|
||||
if (queryResponse.IsSuccessStatusCode)
|
||||
{
|
||||
string queryBody = await queryResponse.Content.ReadAsStringAsync(probeCt);
|
||||
|
||||
// A valid Prometheus response contains "status":"success" and "resultType".
|
||||
|
||||
if (queryBody.Contains("\"status\":\"success\"", StringComparison.Ordinal) &&
|
||||
queryBody.Contains("\"resultType\"", StringComparison.Ordinal))
|
||||
{
|
||||
logger.LogInformation("Confirmed via PromQL query: {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
||||
return true;
|
||||
}
|
||||
|
||||
logger.LogInformation("/api/v1/query returned 200 but body was: {Body}", queryBody[..Math.Min(200, queryBody.Length)]);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 3: Try /-/healthy as a last resort.
|
||||
|
||||
Uri healthyUri = new($"{proxyBase}/-/healthy");
|
||||
|
||||
using (HttpRequestMessage healthyRequest = new(HttpMethod.Get, healthyUri))
|
||||
{
|
||||
using HttpResponseMessage healthyResponse = await httpClient.SendAsync(healthyRequest, probeCt);
|
||||
|
||||
logger.LogInformation("Probe {Service}:{Port} in {Namespace} /-/healthy => {StatusCode}",
|
||||
serviceName, port, ns, (int)healthyResponse.StatusCode);
|
||||
|
||||
if (healthyResponse.IsSuccessStatusCode)
|
||||
{
|
||||
string healthyBody = await healthyResponse.Content.ReadAsStringAsync(probeCt);
|
||||
|
||||
if (healthyBody.Contains("Prometheus", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
logger.LogInformation("Confirmed via /-/healthy: {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
||||
return true;
|
||||
}
|
||||
|
||||
logger.LogInformation("/-/healthy returned 200 but body was: {Body}", healthyBody.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
// Per-probe timeout — service didn't respond fast enough, skip it.
|
||||
logger.LogInformation("Timeout probing {Service}:{Port} in {Namespace} (>{TimeoutSeconds}s)",
|
||||
serviceName, port, ns, ProbeTimeout.TotalSeconds);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
// Service is not reachable via proxy.
|
||||
logger.LogInformation("HTTP error probing {Service}:{Port} in {Namespace}: {Message}", serviceName, port, ns, ex.Message);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Unexpected error probing {Service}:{Port} in {Namespace}", serviceName, port, ns);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a Kubernetes client from the cluster's stored kubeconfig,
|
||||
/// selecting the specific context that was chosen during registration.
|
||||
/// </summary>
|
||||
private static Kubernetes BuildClient(KubernetesCluster cluster)
|
||||
{
|
||||
// Write kubeconfig to a temporary stream for parsing.
|
||||
|
||||
using MemoryStream stream = new(Encoding.UTF8.GetBytes(cluster.KubeConfig));
|
||||
KubernetesClientConfiguration config = KubernetesClientConfiguration.BuildConfigFromConfigFile(
|
||||
stream,
|
||||
currentContext: cluster.ContextName);
|
||||
|
||||
return new Kubernetes(config);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user